.gitlab-ci.yml

stages:
  - setup
  - builds
  - run
  - tests
  - cleanup

# JOB_DIR points to a persistent working space used for most stages in this pipeline but
# that is unique to this pipeline.
# We use the "fetch" strategy to speed up the startup of stages
variables:
  JOB_DIR: "/gpfs/f5/gfdl_o/scratch/oar.gfdl.mom6-account/runner/builds/$CI_PIPELINE_ID"
  GIT_STRATEGY: fetch

# Always eport value of $JOB_DIR
before_script:
  - echo Job directory set to $JOB_DIR

# Test that merge with dev/gfdl works.
p:merge:
  stage: setup
  tags:
    - mom6-ci-c5
  script:
    - git pull --no-edit https://github.com/NOAA-GFDL/MOM6.git dev/gfdl

# Setup the persistent JOB_DIR for all subsequent stages
#
# This basically setups up a complete tree much as a user would in their workflow
p:clone:
  stage: setup
  tags:
    - mom6-ci-c5
  script:
    # NOTE: We could sweep any builds older than 3 days here if needed
    #- find $HOME/ci/[0-9]* -mtime +3 -delete 2> /dev/null || true
    - .gitlab/pipeline-ci-tool.sh create-job-dir
#.gitlab/pipeline-ci-tool.sh clean-job-dir

# Make work spaces for running simultaneously in parallel jobs
#
# Each work space is a clone of MOM6-examples with symbolic links for the build and data directories
# so they can share executables which can run simultaneously without interfering with each other

s:work-space:pgi:
  stage: setup
  tags:
    - mom6-ci-c5
  needs: ["p:clone"]
  script:
    - .gitlab/pipeline-ci-tool.sh copy-test-space pgi

s:work-space:intel:
  stage: setup
  tags:
    - mom6-ci-c5
  needs: ["p:clone"]
  script:
    - .gitlab/pipeline-ci-tool.sh copy-test-space intel

s:work-space:gnu:
  stage: setup
  tags:
    - mom6-ci-c5
  needs: ["p:clone"]
  script:
    - .gitlab/pipeline-ci-tool.sh copy-test-space gnu

s:work-space:gnu-restarts:
  stage: setup
  tags:
    - mom6-ci-c5
  needs: ["p:clone"]
  script:
    - .gitlab/pipeline-ci-tool.sh copy-test-space gnu-rst

# Compile executables
#
# gnu:repro, gnu:debug, intel:repro and pgi:repro are used by their respective run:* jobs
# gnu:ice-only-nolib and gnu:ocean-only-nolibs are not used but simply test that the model compiles without libraries

compile:pgi:repro:
  stage: builds
  needs: ["p:clone"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh mrs-compile repro_pgi

compile:intel:repro:
  stage: builds
  needs: ["p:clone"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh mrs-compile repro_intel

compile:gnu:repro:
  stage: builds
  needs: ["p:clone"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh mrs-compile repro_gnu mrs-compile static_gnu

compile:gnu:debug:
  stage: builds
  needs: ["p:clone"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh mrs-compile debug_gnu

compile:gnu:ocean-only-nolibs:
  stage: builds
  needs: ["p:clone"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh nolibs-ocean-only-compile gnu

compile:gnu:ice-ocean-nolibs:
  stage: builds
  needs: ["p:clone"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh nolibs-ocean-ice-compile gnu

# Runs

run:pgi:
  stage: run
  needs: ["s:work-space:pgi","compile:pgi:repro"]
  tags:
    - mom6-ci-c5
  script:
    - sbatch --clusters=c5 --nodes=12 --time=${MOM6_RUN_JOB_DURATION:=15:00} --account=gfdl_o --qos=debug --job-name=mom6_pgi_tests --output=log.$CI_JOB_ID --wait .gitlab/pipeline-ci-tool.sh run-suite pgi SNL && ( egrep -v 'pagefaults|HiWaterMark=' log.$CI_JOB_ID ; echo Job returned normally ) || ( cat log.$CI_JOB_ID ; echo Job failed ; exit 911 )
    - test -f $JOB_DIR/CI-BATCH-SUCCESS-pgi-SNL || ( echo Batch job did not complete ; exit 911 )

run:intel:
  stage: run
  needs: ["s:work-space:intel","compile:intel:repro"]
  tags:
    - mom6-ci-c5
  script:
    - sbatch --clusters=c5 --nodes=12 --time=${MOM6_RUN_JOB_DURATION:=15:00} --account=gfdl_o --qos=debug --job-name=mom6_intel_tests --output=log.$CI_JOB_ID --wait .gitlab/pipeline-ci-tool.sh run-suite intel SNL && ( egrep -v 'pagefaults|HiWaterMark=' log.$CI_JOB_ID ; echo Job returned normally ) || ( cat log.$CI_JOB_ID ; echo Job failed ; exit 911 )
    - test -f $JOB_DIR/CI-BATCH-SUCCESS-intel-SNL || ( echo Batch job did not complete ; exit 911 )

run:gnu:
  stage: run
  needs: ["s:work-space:gnu","compile:gnu:repro","compile:gnu:debug"]
  tags:
    - mom6-ci-c5
  script:
    - sbatch --clusters=c5 --nodes=12 --time=${MOM6_RUN_JOB_DURATION:=15:00} --account=gfdl_o --qos=debug --job-name=mom6_gnu_tests --output=log.$CI_JOB_ID --wait .gitlab/pipeline-ci-tool.sh run-suite gnu SNLDT && ( egrep -v 'pagefaults|HiWaterMark=' log.$CI_JOB_ID ; echo Job returned normally ) || ( cat log.$CI_JOB_ID ; echo Job failed ; exit 911 )
    - test -f $JOB_DIR/CI-BATCH-SUCCESS-gnu-SNLDT || ( echo Batch job did not complete ; exit 911 )

run:gnu-restarts:
  stage: run
  needs: ["s:work-space:gnu-restarts","compile:gnu:repro"]
  tags:
    - mom6-ci-c5
  script:
    - sbatch --clusters=c5 --nodes=12 --time=${MOM6_RUN_JOB_DURATION:=15:00} --account=gfdl_o --qos=debug --job-name=mom6_gnu_restarts --output=log.$CI_JOB_ID --wait .gitlab/pipeline-ci-tool.sh run-suite gnu R && ( egrep -v 'pagefaults|HiWaterMark=' log.$CI_JOB_ID ; echo Job returned normally ) || ( cat log.$CI_JOB_ID ; echo Job failed ; exit 911 )
    - test -f $JOB_DIR/CI-BATCH-SUCCESS-gnu-R || ( echo Batch job did not complete ; exit 911 )

# GH/autoconf tests (duplicates the GH actions tests)
#
# These stages replace the "before_script" and so start in the transient work-space provided by gitlab.
# We work here to avoid collisions with parallel jobs

actions:gnu:
  stage: tests
  needs: []
  tags:
    - mom6-ci-c5
  before_script:
    - echo -e "\e[0Ksection_start:`date +%s`:submodules[collapsed=true]\r\e[0KCloning submodules"
    - git submodule init ; git submodule update
    - echo -e "\e[0Ksection_end:`date +%s`:submodules\r\e[0K"
  script:
    - echo -e "\e[0Ksection_start:`date +%s`:compile[collapsed=true]\r\e[0KCompiling executables"
    - cd .testing
    - module unload darshan-runtime intel PrgEnv-intel ; module load PrgEnv-gnu/8.5.0 cray-hdf5 cray-netcdf ; module switch gcc-native/12.3
    - FC=ftn MPIFC=ftn CC=cc  make -s -j
    - MPIRUN= FC=ftn MPIFC=ftn CC=cc make preproc -s -j
    - echo -e "\e[0Ksection_end:`date +%s`:compile\r\e[0K"
    - (echo '#!/bin/bash';echo 'make MPIRUN="srun -mblock --exclusive" test -s -j') > job.sh
    - sbatch --clusters=c5 --nodes=2 --time=0:10:00 --account=gfdl_o --qos=debug --job-name=MOM6.gnu.testing --output=log.$CI_JOB_ID --wait job.sh || ( cat log.$CI_JOB_ID ; exit 911 ) && make test -s
    - make test.summary

actions:intel:
  stage: tests
  needs: []
  tags:
    - mom6-ci-c5
  before_script:
    - echo -e "\e[0Ksection_start:`date +%s`:submodules[collapsed=true]\r\e[0KCloning submodules"
    - git submodule init ; git submodule update
    - echo -e "\e[0Ksection_end:`date +%s`:submodules\r\e[0K"
  script:
    - echo -e "\e[0Ksection_start:`date +%s`:compile[collapsed=true]\r\e[0KCompiling executables"
    - cd .testing
    - module unload darshan-runtime ; module unload intel cray-libsci cray-mpich PrgEnv-intel ; module load PrgEnv-intel intel/2023.2.0 cray-hdf5 cray-netcdf cray-mpich
    - FC=ftn MPIFC=ftn CC=cc make -s -j
    - MPIRUN= FC=ftn MPIFC=ftn CC=cc make preproc -s -j
    - echo -e "\e[0Ksection_end:`date +%s`:compile\r\e[0K"
    - (echo '#!/bin/bash';echo 'make MPIRUN="srun -mblock --exclusive" test -s -j') > job.sh
    - sbatch --clusters=c5 --nodes=2 --time=0:10:00 --account=gfdl_o --qos=debug --job-name=MOM6.intel.testing --output=log.$CI_JOB_ID --wait job.sh || ( cat log.$CI_JOB_ID ; exit 911 ) && make test -s
    - make test.summary

# Tests
#
# stats file tests involve comparing the check sums of the generated files against the check sums in the stats-repo
# log file tests involve comparing the check sums of the generated files against the check sums in MOM6-examples

t:pgi:symmetric:
  stage: tests
  needs: ["run:pgi"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats pgi S

t:pgi:non-symmetric:
  stage: tests
  needs: ["run:pgi"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats pgi N

t:pgi:layout:
  stage: tests
  needs: ["run:pgi"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats pgi L

t:pgi:params:
  stage: tests
  needs: ["run:pgi"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh check-params pgi
  allow_failure: true

t:intel:symmetric:
  stage: tests
  needs: ["run:intel"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats intel S

t:intel:non-symmetric:
  stage: tests
  needs: ["run:intel"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats intel N

t:intel:layout:
  stage: tests
  needs: ["run:intel"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats intel L

t:intel:params:
  stage: tests
  needs: ["run:intel"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh check-params intel
  allow_failure: true

t:gnu:symmetric:
  stage: tests
  needs: ["run:gnu"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats gnu S

t:gnu:non-symmetric:
  stage: tests
  needs: ["run:gnu"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats gnu N

t:gnu:layout:
  stage: tests
  needs: ["run:gnu"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats gnu L

t:gnu:static:
  stage: tests
  needs: ["run:gnu"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats gnu T

t:gnu:symmetric-debug:
  stage: tests
  needs: ["run:gnu"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats gnu D

t:gnu:restart:
  stage: tests
  needs: ["run:gnu-restarts"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh check-stats gnu R

t:gnu:params:
  stage: tests
  needs: ["run:gnu"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh check-params gnu
  allow_failure: true

t:gnu:diags:
  stage: tests
  needs: ["run:gnu"]
  tags:
    - mom6-ci-c5
  script:
    - .gitlab/pipeline-ci-tool.sh check-diags gnu
  allow_failure: true

# We cleanup ONLY if the preceding stages were completed successfully
cleanup:
  stage: cleanup
  tags:
    - mom6-ci-c5
  before_script:
    - echo Skipping usual preamble
  script:
    - rm -rf $HOME/ci/$CI_PIPELINE_ID
    - rm -rf $JOB_DIR