Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jenkins updates #2334

Merged
Merged
Changes from all commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
bd9ed1f
removed env.MACHINE_ID=machine in Jenkins, same as export MACHINE_ID=…
Feb 14, 2024
fa9990d
Update build_all.sh
TerrenceMcGuinness-NOAA Feb 14, 2024
6901c9c
used try catch over run experments get archive of error logs and runn…
Feb 15, 2024
6991208
put HOMEgfs back in the paths to the CI scripts
Feb 15, 2024
481e407
trying recusive scm
Feb 15, 2024
3d83f36
just hadded buildDiscarder line
Feb 15, 2024
5840fac
needed w in build all for new tests
Feb 16, 2024
8ac6b09
Merge branch 'NOAA-EMC:develop' into hotfix_pipeline_no-MACHINE_ID
TerrenceMcGuinness-NOAA Feb 16, 2024
1698ff8
-w is not really in develop now removeing w from yamls build and remo…
Feb 16, 2024
63a3e4f
Merge branch 'hotfix_pipeline_no-MACHINE_ID' of https://github.com/Te…
Feb 16, 2024
9e5a9df
Merge branch 'develop' into hotfix_pipeline_no-MACHINE_ID
Feb 16, 2024
8066167
testing a method for implenting skip host clause
Feb 16, 2024
4229ab5
testing a method for implenting skip host clause
Feb 16, 2024
4473d96
add python script to just check case skip host
Feb 16, 2024
646002e
added check case test in python for return false if skip_ci_on_hosts …
Feb 16, 2024
e572cde
added python code to check for skip case on host
Feb 16, 2024
d111f94
check case script now returns true to run and false not to run
Feb 16, 2024
af3f5e0
put in test for skip cases with when expression with env var
Feb 16, 2024
e8cc2ba
make seperate stage to check skip
Feb 16, 2024
e108f18
add wxflow link in utils dir and another case in test list
Feb 16, 2024
f5be771
add env.HOME to point to check script
Feb 16, 2024
7c3d377
got imports to work by adding links to modules dirs
Feb 16, 2024
ea3c1ec
added echo for result of check case
Feb 16, 2024
0cd27da
Merge branch 'hotfix_pipeline_no-MACHINE_ID' of https://github.com/Te…
Feb 16, 2024
ea338b8
testing list again
Feb 16, 2024
fb0e406
spelled global variable test_cases_list wrong
Feb 16, 2024
37bf83b
get all cases for host by checking for skips
Feb 16, 2024
6b32939
added get cases list of cases that are not skipped for that host
Feb 16, 2024
1aa8d2a
switch list to ones that do run on host from ones that dont
Feb 17, 2024
371d4f6
misspelled caseList and fixed
Feb 17, 2024
c3d0734
added skip host orion for C96_atmos snow DA
Feb 20, 2024
4b937ed
Merge branch 'NOAA-EMC:develop' into jenkins_updates
TerrenceMcGuinness-NOAA Feb 21, 2024
6fa270f
trying to reduce workspace paths
Feb 21, 2024
7f63c5e
just run bad actor case for datapath issue
Feb 21, 2024
1fcfc4c
just changed numToKeep
Feb 21, 2024
25cb1cb
missed bracket on agent
Feb 21, 2024
da7bc97
used ws for custom instead
Feb 21, 2024
afd77b9
took out in label block
Feb 21, 2024
689e0ee
spell custom_workspace
Feb 21, 2024
56b51b6
added common workspace with miminum nubmer of chars to path
Feb 22, 2024
466ba83
added common workspace with miminum nubmer of chars to path
Feb 22, 2024
3baed5f
put all tests back and removed Keep limiter
Feb 22, 2024
93ff847
python norms needed extra line delteted in get_host_case_list
Feb 22, 2024
ab28fca
added host name to argument to get_host_case_list.py
Feb 23, 2024
fda0610
fixing python norms with removing white spaces
Feb 23, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 31 additions & 20 deletions Jenkinsfile
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
def Machine = 'none'
def machine = 'none'
def HOME = 'none'
def localworkspace = 'none'
def commonworkspace = 'none'
def caseList = ''
def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES']

pipeline {
agent { label 'built-in' }
@@ -20,7 +20,6 @@ pipeline {
agent { label 'built-in' }
steps {
script {
localworkspace = env.WORKSPACE
machine = 'none'
for (label in pullRequest.labels) {
echo "Label: ${label}"
@@ -42,14 +41,16 @@ pipeline {
agent { label "${machine}-emc" }
steps {
script {
properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
HOME = "${WORKSPACE}/TESTDIR"
commonworkspace = "${WORKSPACE}"
sh(script: "mkdir -p ${HOME}/RUNTESTS")
pullRequest.addLabel("CI-${Machine}-Building")
if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) {
pullRequest.removeLabel("CI-${Machine}-Ready")
ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
HOME = "${WORKSPACE}"
sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/error.logs")
pullRequest.addLabel("CI-${Machine}-Building")
if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Ready") }) {
pullRequest.removeLabel("CI-${Machine}-Ready")
}
}
pullRequest.comment("Building and running on ${Machine} in directory ${HOME}")
}
}
}
@@ -73,17 +74,15 @@ pipeline {
def HOMEgfs = "${HOME}/${system}" // local HOMEgfs is used to build the system on per system basis under the common workspace HOME
sh(script: "mkdir -p ${HOMEgfs}")
ws(HOMEgfs) {
env.MACHINE_ID = machine // MACHINE_ID is used in the build scripts to determine the machine and is added to the shell environment
if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore
pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}<br>Still doing a checkout to get the latest changes")
sh(script: 'source workflow/gw_setup.sh; git pull --recurse-submodules')
checkout scm
dir('sorc') {
sh(script: './link_workflow.sh')
}
} else {
checkout scm
sh(script: 'source workflow/gw_setup.sh;which git;git --version;git submodule update --init --recursive')
def builds_file = readYaml file: 'ci/cases/yamls/build.yaml'
def build_args_list = builds_file['builds']
def build_args = build_args_list[system].join(' ').trim().replaceAll('null', '')
@@ -99,6 +98,9 @@ pipeline {
}
pullRequest.addLabel("CI-${Machine}-Running")
}
if (system == 'gfs') {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why only GFS? Or is this just a way to only do it once?

caseList = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
}
}
}
}
@@ -114,11 +116,15 @@ pipeline {
axis {
name 'Case'
// TODO add dynamic list of cases from env vars (needs addtional plugins)
values 'C48_ATM', 'C48_S2SWA_gefs', 'C48_S2SW', 'C96_atm3DVar', 'C96C48_hybatmDA', 'C96_atmsnowDA' // 'C48mx500_3DVarAOWCDA'
values 'C48C48_ufs_hybatmDA', 'C48_ATM', 'C48_S2SW', 'C48_S2SWA_gefs', 'C48mx500_3DVarAOWCDA', 'C96C48_hybatmDA', 'C96_atm3DVar', 'C96_atmsnowDA'
}
}
stages {
stage('Create Experiment') {

stage('Create Experiments') {
when {
expression { return caseList.contains(Case) }
}
steps {
script {
sh(script: "sed -n '/{.*}/!p' ${HOME}/gfs/ci/cases/pr/${Case}.yaml > ${HOME}/gfs/ci/cases/pr/${Case}.yaml.tmp")
@@ -130,15 +136,20 @@ pipeline {
}
}
}

stage('Run Experiments') {
when {
expression { return caseList.contains(Case) }
}
steps {
script {
HOMEgfs = "${HOME}/gfs" // common HOMEgfs is used to launch the scripts that run the experiments
ws(HOMEgfs) {
pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim()
// pullRequest.comment("**Running** experiment: ${Case} on ${Machine}<br>With the experiment in directory:<br>`${HOME}/RUNTESTS/${pslot}`")
err = sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}")
if (err != 0) {
pullRequest.comment("**Running** experiment: ${Case} on ${Machine}<br>With the experiment in directory:<br>`${HOME}/RUNTESTS/${pslot}`")
try {
sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}")
} catch (Exception e) {
pullRequest.comment("**FAILURE** running experiment: ${Case} on ${Machine}")
sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_all_batch_jobs ${HOME}/RUNTESTS")
ws(HOME) {
@@ -153,12 +164,12 @@ pipeline {
}
error("Failed to run experiments ${Case} on ${Machine}")
}
// pullRequest.comment("**SUCCESS** running experiment: ${Case} on ${Machine}")
}
pullRequest.comment("**SUCCESS** running experiment: ${Case} on ${Machine}")
}

}
}

}
}
}
32 changes: 32 additions & 0 deletions ci/scripts/utils/get_host_case_list.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env python3
import os
from os.path import basename, splitext
import sys
import glob
from wxflow import parse_j2yaml
from wxflow import AttrDict

_here = os.path.dirname(__file__)
_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../..'))

if __name__ == '__main__':

if len(sys.argv) < 2:
print('Usage: get_host_case_list.py <host_name>')
sys.exit(1)

host = sys.argv[1]

case_list = []
HOMEgfs = _top
data = AttrDict(HOMEgfs=_top)
data.update(os.environ)

case_files = glob.glob(f'{HOMEgfs}/ci/cases/pr/*.yaml')
for case_yaml in case_files:
case_conf = parse_j2yaml(path=case_yaml, data=data)
if 'skip_ci_on_hosts' in case_conf:
if host.lower() in [machine.lower() for machine in case_conf.skip_ci_on_hosts]:
continue
case_list.append(splitext(basename(case_yaml))[0])
print(' '.join(case_list))
1 change: 1 addition & 0 deletions ci/scripts/utils/wxflow