Merge branch 'NOAA-EMC:develop' into feature/gmi_on

.github/workflows/pw_aws_centos.yaml → .github/workflows/pw_aws_ci.yaml

@@ -1,7 +1,4 @@
-name: gw-ci-aws-centos
-
-on: [workflow_dispatch]
-
+name: gw-ci-aws
 # TEST_DIR contains 2 directories;
 # 1. HOMEgfs: clone of the global-workflow
 # 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
@@ -10,33 +7,73 @@ on: [workflow_dispatch]
 # ├── HOMEgfs
 # └── RUNTESTS
 #     ├── COMROT
-#     │   └── ${pslot}
+#     │   └── ${pslot}
 #     └── EXPDIR
 #         └── ${pslot}
+
+on:
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: 'Pull Request Number (use 0 for non-PR)'
+        required: true
+        default: '0'
+      os:
+        description: 'Operating System'
+        required: true
+        type: choice
+        options:
+          - rocky
+          - centos
+
 env:
   TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
   MACHINE_ID: noaacloud
 
 jobs:
+  fetch-branch:
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ secrets.GITHUBTOKEN }}
+    outputs:
+      branch: ${{ steps.get-branch.outputs.branch }}
+    steps:
+    - name: Fetch branch name for PR
+      id: get-branch
+      run: |
+        pr_number=${{ github.event.inputs.pr_number }}
+        repo=${{ github.repository }}
+        if [ "$pr_number" -eq "0" ]; then
+          branch=${{ github.event.inputs.ref }}
+        else
+          branch=$(gh pr view $pr_number --repo $repo --json headRefName --jq '.headRefName')
+        fi
+        echo "::set-output name=branch::$branch"
+
   checkout:
-    runs-on: [self-hosted, aws, parallelworks, centos]
+    needs: fetch-branch
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     timeout-minutes: 600
-
     steps:
-
     - name: Checkout global-workflow
       uses: actions/checkout@v4
       with:
         path: ${{ github.run_id }}/HOMEgfs
         submodules: 'recursive'
-        ref: ${{ github.event.pull_request.head.ref }}
+        ref: ${{ needs.fetch-branch.outputs.branch }}
 
   build-link:      
-    runs-on: [self-hosted, aws, parallelworks, centos]
     needs: checkout
-
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     steps:
-
     - name: Build components
       run: |
         cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
@@ -48,12 +85,15 @@ jobs:
         ./link_workflow.sh
 
   create-experiments:
-    needs: checkout
-    runs-on: [self-hosted, aws, parallelworks, centos]
+    needs: build-link
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     strategy:
       matrix:
         case: ["C48_ATM"]
-
     steps:
       - name: Create Experiments ${{ matrix.case }}
         env:
@@ -68,9 +108,12 @@ jobs:
 
   run-experiments:
     needs: create-experiments
-    runs-on: [self-hosted, aws, parallelworks, centos]
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     strategy:
-      max-parallel: 2
       matrix:
         case: ["C48_ATM"]
     steps:
@@ -81,9 +124,13 @@ jobs:
 
   clean-up:
     needs: run-experiments
-    runs-on: [self-hosted, aws, parallelworks, centos]
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     steps:
-      - name: Clean-up
+      - name: Clean up workspace
         run: |
-          cd ${{ github.workspace }}
-          rm -rf ${{ github.run_id }}
+          echo "Cleaning up workspace"
+          rm -rf ${{ env.TEST_DIR }}

env/HERA.env

@@ -9,7 +9,7 @@ fi
 
 step=$1
 
-export launcher="srun -l --export=ALL"
+export launcher="srun -l --export=ALL --hint=nomultithread"
 export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 
 #export POSTAMBLE_CMD='report-mem'
@@ -50,7 +50,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
     export POE="NO"
     export BACK="NO"
     export sys_tp="HERA"
-    export launcher_PREP="srun"
+    export launcher_PREP="srun --hint=nomultithread"
 
 elif [[ "${step}" = "prepsnowobs" ]]; then
 

env/HERCULES.env

@@ -9,7 +9,7 @@ fi
 
 step=$1
 
-export launcher="srun -l --export=ALL"
+export launcher="srun -l --export=ALL --hint=nomultithread"
 export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 
 # Configure MPI environment
@@ -48,7 +48,7 @@ case ${step} in
     export POE="NO"
     export BACK=${BACK:-"YES"}
     export sys_tp="HERCULES"
-    export launcher_PREP="srun"
+    export launcher_PREP="srun --hint=nomultithread"
  ;;
  "prepsnowobs")
 

env/ORION.env

@@ -9,7 +9,7 @@ fi
 
 step=$1
 
-export launcher="srun -l --export=ALL"
+export launcher="srun -l --export=ALL --hint=nomultithread"
 export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 
 # Configure MPI environment
@@ -45,7 +45,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
     export POE="NO"
     export BACK=${BACK:-"YES"}
     export sys_tp="ORION"
-    export launcher_PREP="srun"
+    export launcher_PREP="srun --hint=nomultithread"
 
 elif [[ "${step}" = "prepsnowobs" ]]; then
 

parm/config/gefs/config.efcs

@@ -26,18 +26,6 @@ source "${EXPDIR}/config.ufs" ${string}
 # Get task specific resources
 source "${EXPDIR}/config.resources" efcs
 
-# nggps_diag_nml
-export FHOUT=${FHOUT_ENKF:-3}
-if [[ "${RUN}" == "enkfgfs" ]]; then
-    export FHOUT=${FHOUT_ENKF_GFS:-${FHOUT}}
-fi
-
-# model_configure
-export FHMAX=${FHMAX_ENKF:-9}
-if [[ "${RUN}" == "enkfgfs" ]]; then
-   export FHMAX=${FHMAX_ENKF_GFS:-${FHMAX}}
-fi
-
 # Stochastic physics parameters (only for ensemble forecasts)
 export DO_SKEB="YES"
 export SKEB="0.8,-999,-999,-999,-999"
@@ -74,6 +62,6 @@ if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
 else
     export ODA_INCUPD="False"
 fi
-export restart_interval="${restart_interval_enkfgfs:-12}"
+export restart_interval="${restart_interval_gfs:-12}"
 
 echo "END: config.efcs"

parm/config/gefs/config.fcst

@@ -35,7 +35,8 @@ IFS=', ' read -ra segments <<< "${FCST_SEGMENTS}"
 # Determine MIN and MAX based on the forecast segment
 export FHMIN=${segments[${FCST_SEGMENT}]}
 export FHMAX=${segments[${FCST_SEGMENT}+1]}
-# Cap other FHMAX variables at FHMAX for the segment
+# Cap other FH variables at FHMAX for the segment
+export FHMIN_WAV=$(( FHMIN > FHMIN_WAV ? FHMIN : FHMIN_WAV ))
 export FHMAX_HF=$(( FHMAX_HF_GFS > FHMAX ? FHMAX : FHMAX_HF_GFS ))
 export FHMAX_WAV=$(( FHMAX_WAV > FHMAX ? FHMAX : FHMAX_WAV ))
 # shellcheck disable=SC2153

parm/config/gfs/config.fcst

@@ -38,7 +38,8 @@ case ${RUN} in
     # Determine MIN and MAX based on the forecast segment
     export FHMIN=${segments[${FCST_SEGMENT}]}
     export FHMAX=${segments[${FCST_SEGMENT}+1]}
-    # Cap other FHMAX variables at FHMAX for the segment
+    # Cap other FH variables at FHMAX for the segment
+    export FHMIN_WAV=$(( FHMIN > FHMIN_WAV ? FHMIN : FHMIN_WAV ))
     export FHMAX_HF=$(( FHMAX_HF_GFS > FHMAX ? FHMAX : FHMAX_HF_GFS ))
     export FHMAX_WAV=$(( FHMAX_WAV > FHMAX ? FHMAX : FHMAX_WAV ))
     # shellcheck disable=SC2153

parm/stage/wave.yaml.j2

@@ -9,5 +9,9 @@ wave:
         {% for mem in range(first_mem, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_WAVE_RESTART_PREV_MEM = COMOUT_WAVE_RESTART_PREV_MEM_list[imem] %}
-        - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.restart.{{ waveGRD }}", "{{ COMOUT_WAVE_RESTART_PREV_MEM }}"]
+        {% if path_exists(ICSDIR ~ "/" ~ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) ~ "/" ~ m_prefix ~ ".restart." ~ waveGRD) %}
+        - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.restart.{{ waveGRD }}", "{{ COMOUT_WAVE_RESTART_PREV_MEM }}/{{ m_prefix }}.restart.ww3"]
+        {% else %}
+        - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.restart.ww3", "{{ COMOUT_WAVE_RESTART_PREV_MEM }}"]
+        {% endif %}
         {% endfor %} # mem loop

ush/forecast_det.sh

@@ -93,12 +93,9 @@ UFS_det(){
 
     # Check for WW3 restart availability
     if [[ "${cplwav}" == ".true." ]]; then
-      local ww3_grid
-      for ww3_grid in ${waveGRD} ; do
-        if [[ ! -f "${DATArestart}/WW3_RESTART/${rdate:0:8}.${rdate:8:2}0000.restart.${ww3_grid}" ]]; then
-          ww3_rst_ok="NO"
-        fi
-      done
+      if [[ ! -f "${DATArestart}/WW3_RESTART/${rdate:0:8}.${rdate:8:2}0000.restart.ww3" ]]; then
+        ww3_rst_ok="NO"
+      fi
     fi
 
     # Collective check

ush/forecast_postdet.sh

@@ -326,7 +326,7 @@ FV3_out() {
 WW3_postdet() {
   echo "SUB ${FUNCNAME[0]}: Linking input data for WW3"
 
-  local ww3_grid
+  local ww3_grid first_ww3_restart_out ww3_restart_file
   # Copy initial condition files:
   if [[ "${warm_start}" == ".true." ]]; then
     local restart_date restart_dir
@@ -338,29 +338,35 @@ WW3_postdet() {
       restart_dir="${COMIN_WAVE_RESTART_PREV}"
     fi
     echo "Copying WW3 restarts for 'RUN=${RUN}' at '${restart_date}' from '${restart_dir}'"
-    local ww3_restart_file
-    for ww3_grid in ${waveGRD} ; do
-      ww3_restart_file="${restart_dir}/${restart_date:0:8}.${restart_date:8:2}0000.restart.${ww3_grid}"
-      if [[ ! -f "${ww3_restart_file}" ]]; then
-        echo "WARNING: WW3 restart file '${ww3_restart_file}' not found for warm_start='${warm_start}', will start from rest!"
-        if [[ "${RERUN}" == "YES" ]]; then
-          # In the case of a RERUN, the WW3 restart file is required
-          echo "FATAL ERROR: WW3 restart file '${ww3_restart_file}' not found for RERUN='${RERUN}', ABORT!"
-          exit 1
-        fi
-      fi
-      if [[ "${waveMULTIGRID}" == ".true." ]]; then
-        ${NCP} "${ww3_restart_file}" "${DATA}/restart.${ww3_grid}" \
-        || ( echo "FATAL ERROR: Unable to copy WW3 IC, ABORT!"; exit 1 )
+    ww3_restart_file="${restart_dir}/${restart_date:0:8}.${restart_date:8:2}0000.restart.ww3"
+    if [[ -f "${ww3_restart_file}" ]]; then
+      ${NCP} "${ww3_restart_file}" "${DATA}/restart.ww3" \
+      || ( echo "FATAL ERROR: Unable to copy WW3 IC, ABORT!"; exit 1 )
+    else
+      if [[ "${RERUN}" == "YES" ]]; then
+        # In the case of a RERUN, the WW3 restart file is required
+        echo "FATAL ERROR: WW3 restart file '${ww3_restart_file}' not found for RERUN='${RERUN}', ABORT!"
+        exit 1
       else
-        ${NCP} "${ww3_restart_file}" "${DATA}/restart.ww3" \
-        || ( echo "FATAL ERROR: Unable to copy WW3 IC, ABORT!"; exit 1 )
+        echo "WARNING: WW3 restart file '${ww3_restart_file}' not found for warm_start='${warm_start}', will start from rest!"
       fi
-    done
+    fi
+
+    first_ww3_restart_out=$(date --utc -d "${restart_date:0:8} ${restart_date:8:2} + ${restart_interval} hours" +%Y%m%d%H)
   else  # cold start
     echo "WW3 will start from rest!"
+    first_ww3_restart_out="${model_start_date_current_cycle}"
   fi  # [[ "${warm_start}" == ".true." ]]
 
+  # Link restart files
+  local ww3_restart_file
+  # Use restart_date if it was determined above, otherwise use initialization date
+  for (( vdate = first_ww3_restart_out; vdate <= forecast_end_cycle;
+         vdate = $(date --utc -d "${vdate:0:8} ${vdate:8:2} + ${restart_interval} hours" +%Y%m%d%H) )); do
+    ww3_restart_file="${vdate:0:8}.${vdate:8:2}0000.restart.ww3"
+    ${NLN} "${DATArestart}/WW3_RESTART/${ww3_restart_file}" "${ww3_restart_file}"
+  done
+
   # Link output files
   local wavprfx="${RUN}wave${WAV_MEMBER:-}"
   if [[ "${waveMULTIGRID}" == ".true." ]]; then

ush/forecast_predet.sh

@@ -556,10 +556,10 @@ WW3_predet(){
   echo "SUB ${FUNCNAME[0]}: WW3 before run type determination"
 
   if [[ ! -d "${COMOUT_WAVE_HISTORY}" ]]; then mkdir -p "${COMOUT_WAVE_HISTORY}"; fi
-  if [[ ! -d "${COMOUT_WAVE_RESTART}" ]]; then mkdir -p "${COMOUT_WAVE_RESTART}" ; fi
+  if [[ ! -d "${COMOUT_WAVE_RESTART}" ]]; then mkdir -p "${COMOUT_WAVE_RESTART}"; fi
 
-  if [[ ! -d "${DATArestart}/WAVE_RESTART" ]]; then mkdir -p "${DATArestart}/WAVE_RESTART"; fi
-  ${NLN} "${DATArestart}/WAVE_RESTART" "${DATA}/restart_wave"
+  if [[ ! -d "${DATArestart}/WW3_RESTART" ]]; then mkdir -p "${DATArestart}/WW3_RESTART"; fi
+  # Wave restarts are linked in postdet to only create links for files that will be created
 
   # Files from wave prep and wave init jobs
   # Copy mod_def files for wave grids