.gitlab-ci.yml

variables:
  GIT_STRATEGY: clone   # use clone and not fetch otherwise rerunning jobs after a push fails
  CMSSW_GIT_REFERENCE: "${CI_PROJECT_DIR}/cmssw.git"
  TESTDIR: "${CI_PROJECT_DIR}/testdir"
  SCRIPTDIR: "${CI_PROJECT_DIR}/scripts"
  GITHUB_QUIET: 0  #  1 to turn off github posts, 0 otherwise
#@TESTVARS@
# DO NOT DELETE THE TESTVARS COMMENT - gets replaced for each branch with necessary variables
  WEBDIR: "UHH2integration/${LOCALBRANCH}"  # Directory to be copied to EOS_PATH, LOCALBRANCH set in testPR


default:
  image: ${IMAGE}
  tags:
    - cvmfs

before_script:
  # Common part to setup vars
  - source /cvmfs/grid.cern.ch/etc/profile.d/setup-cvmfs-ui.sh
  - source /cvmfs/cms.cern.ch/cmsset_default.sh
  - source ${SCRIPTDIR}/pre_install.sh  # setup general vars
  # Update PRID env var if it wasn't already set & add to vars
  - "[ -z \"$PRID\" ] && python scripts/get_pr_id.py --id $PRNUM --dumpFilename ${SCRIPTDIR}/to_test.sh --dumpVarname PRID && source ${SCRIPTDIR}/to_test.sh"
  - "[ -e ${SCRIPTDIR}/to_test.sh ] && source ${SCRIPTDIR}/to_test.sh"

get-uhh:
  # Get the ref & modified versions of UHH2 that is used in later steps along with diff
  stage: getit
  except:
    - master
  script:
    - source ${SCRIPTDIR}/notify_github.sh "start"
    - source ${SCRIPTDIR}/add_label.sh $REFBRANCH
    - source ${SCRIPTDIR}/get_uhh_ref_and_new.sh || source ${SCRIPTDIR}/notify_github.sh "failed" "Failed to checkout UHH2 & merge PR"
  artifacts:
    paths:
      - UHH2
    expire_in: 2 days

# -----------------------------------------------------------------------------
# BUILD & COMPILE JOBS
# -----------------------------------------------------------------------------
build-new:
  # Build the new user's version
  stage: build
  dependencies:
    - get-uhh
  except:
    - master
  artifacts:
    name: ${CI_COMMIT_REF_NAME}-new-setup
    expire_in: 6 mos
    paths:
      - ${TESTDIR}/all_${CI_COMMIT_REF_NAME}.tgz
      - ${TESTDIR}/deploy_script_all_${CI_COMMIT_REF_NAME}.sh
      - ${CI_PROJECT_DIR}/build-success
  script:
    - yum install -y libtool  # needed for autoreconf when setting up fastjet
    - cd UHH2 && git checkout ${LOCALBRANCH} && cd .. # Use the modified branch
    - mkdir -p ${TESTDIR} && cd ${TESTDIR}
    - uname -a
    # deploy CMSSW, UHH
    - source ${CI_PROJECT_DIR}/UHH2/scripts/install.sh || source ${SCRIPTDIR}/notify_github.sh "failed" "install.sh failed, please check log"
    - source ${SCRIPTDIR}/replace_UHH.sh  # Use the UHH2 version we checked out
    # Compile everything
    - source ${SCRIPTDIR}/post_install.sh || source ${SCRIPTDIR}/notify_github.sh "failed" "Compilation of SFrame / UHH2 failed, please check log"
    - source ${SCRIPTDIR}/post_comment.sh "Compilation with PR successful"
    # Check the python is even valid
    # -tt flag checks tabs vs spaces
    - cd ${CMSSW_BASE}/src/UHH2/core/python
    - for f in ntuple*.py; do echo "$f"; python -tt "$f" || source ${SCRIPTDIR}/notify_github.sh "failed" 'Encountered invalid python in ntuplewriter* scripts' ; done
    # Check the CRAB tarball size
    - source /cvmfs/cms.cern.ch/common/crab-setup.sh; cd ${CMSSW_BASE}/src/UHH2/scripts/crab;
      python ${SCRIPTDIR}/makeCheckCrabTarball.py --output testTarball.tgz || source ${SCRIPTDIR}/post_comment.sh "CRAB tarball creation failed or exceeded size limit"; rm testTarball.tgz
    # Tar up CMSSW & SFrame but only for 10*X jobs where we actually run cmsrun jobs
    - if [[ $REFBRANCH == "RunII_10"* ]]; then cd ${TESTDIR} && python ${SCRIPTDIR}/tarballEntireRelease.py -o ${TESTDIR}/all_${CI_COMMIT_REF_NAME}.tgz; fi
    # for later pass/fail check
    - touch ${CI_PROJECT_DIR}/build-success

build-ref:
  # Build the reference UHH version (set in the install.sh in UHH2)
  # so we don't need the replace_UHH.sh
  stage: build
  dependencies:
    - get-uhh
  only:
    variables:
      - $MAKENTUPLES == "1"
  except:
    - master
  artifacts:
    name: ${CI_COMMIT_REF_NAME}-ref-setup
    expire_in: 6 mos
    paths:
      - ${TESTDIR}/all_ref.tgz
      - ${TESTDIR}/deploy_script_all_ref.sh
  script:
    - yum install -y libtool  # needed for autoreconf when setting up fastjet
    - cd UHH2 && git checkout ${REFBRANCH} && cd .. # Use the reference branch
    - mkdir -p ${TESTDIR} && cd ${TESTDIR}
    - source ${CI_PROJECT_DIR}/UHH2/scripts/install.sh # deploy CMSSW, UHH
    - source ${SCRIPTDIR}/post_install.sh  # Compile everything
    - if [[ $REFBRANCH == "RunII_10"* ]]; then cd ${TESTDIR} && python ${SCRIPTDIR}/tarballEntireRelease.py -o ${TESTDIR}/all_ref.tgz; fi  # Tar up CMSSW & SFrame, only if doing cmsrun jobs


# Logic for setting MAKENTUPLES variables is done in openshiftapp.py
# Based on refbranch, and whether or not user included magic words in PR text.
# Note that if you add only/except in the stage iteself, it will ignore this

# version for 102X jobs
.makentuple102-template: &make-ntuples-102
  only:
    variables:
      - $MAKENTUPLES == "1" && $REFBRANCH =~ /.*102X.*/
  except:
    - master

# version for 106X jobs
.makentuple106-template: &make-ntuples-106
  only:
    variables:
      - $MAKENTUPLES == "1" && $REFBRANCH =~ /.*106X.*/
  except:
    - master

# cmsRun job templates
.cmsrun-template: &cmsrun
  stage: cmsrun
  retry: 2
  artifacts:
    name: ${CI_COMMIT_REF_NAME}-${CI_JOB_NAME}
    expire_in: 6 mos
    paths:
      - ${TESTDIR}/Ntuple*.root
      - ${TESTDIR}/log*.txt
      - ${TESTDIR}/*.json
      - ${TESTDIR}/*.awkd

.cmsrun-new-template: &cmsrun-new
  # Run cmsRun on PR
  <<: *cmsrun
  dependencies:
    - build-new

.cmsrun-ref-template: &cmsrun-ref
  # Run cmsRun on reference setup
  # we allow this one to fail, since the PR might be fixing some error in the repo
  <<: *cmsrun
  dependencies:
    - build-ref
  allow_failure: true

# Comparison & webpage job between ntuples
# Must set dependencies manually, including build-new
.compare-webpage-template: &compare-webpage
  stage: compare
  artifacts:
    name: webpage-${CI_JOB_NAME}
    paths:
      - public
      - ${TESTDIR}/*.json
  script:
    # needed to get libSM
    - yum install -y libXext libSM libXrender
    # Deploy our release to get a newer version of python etc, check packages
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt
    # Compare the 2 ntuples, making plots
    - source ${SCRIPTDIR}/makeAllNtupleComparisons.sh
    # generate the webpage here since we have all the necessary inputs
    # (plots, timing json, size json)
    - ls ${TESTDIR}
    - source ${SCRIPTDIR}/makeAllWebpages.sh

# Deploy the pages generated to EOS
# contents actually generated in compare-webpage job
# https://gitlab.cern.ch/gitlabci-examples/deploy_eos/
# requires vars EOS_ACCOUNT_USERNAME, EOS_ACCOUNT_PASSWORD, EOS_PATH
# optionally CI_OUTPUT_DIR (default is public)
# Must set dependencies to compare job
# We do this in separate jobs since the upload is slow, so we want to
# parallelise it as much as possible
.eosdeploy-template: &eos-deploy
  stage: deploy
  # Custom docker image providing the needed tools to deploy in EOS
  image: gitlab-registry.cern.ch/ci-tools/ci-web-deployer:latest
  script:
    # Script that performs the deploy to EOS. Makes use of the variables defined in the project
    - deploy-eos
    # do not run any globally defined before_script or after_script for this step
  before_script: []
  after_script: []

# -----------------------------------------------------------------------------
# CMS RUN JOBS
# -----------------------------------------------------------------------------
# Each {era, type} should have a new job (to run on PR),
# a ref job (to run on current HEAD), a comparison & webpage maker job,
# and a EOS uploader job

cmsrun-2018-data-new:
  # Run 2018 data on PR
  <<: *make-ntuples-102
  <<: *cmsrun-new
  script:
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year 2018 --isData --append "_new"

cmsrun-2018-data-ref:
  # Run 2018 data on reference
  <<: *make-ntuples-102
  <<: *cmsrun-ref
  script:
    - cd ${TESTDIR} && source deploy_script_all_ref.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year 2018 --isData --append "_ref"

compare-webpage-2018-data:
  <<: *make-ntuples-102
  <<: *compare-webpage
  dependencies:
    - build-new
    - cmsrun-2018-data-new
    - cmsrun-2018-data-ref

eosdeploy-2018-data:
  <<: *make-ntuples-102
  <<: *eos-deploy
  dependencies:
    - compare-webpage-2018-data

#-------------------------------------------------------------------------------

cmsrun-2018-mc-new:
  # Run 2018 mc on PR
  <<: *make-ntuples-102
  <<: *cmsrun-new
  script:
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year 2018 --append "_new"

cmsrun-2018-mc-ref:
  # Run 2018 mc on reference
  <<: *make-ntuples-102
  <<: *cmsrun-ref
  script:
    - cd ${TESTDIR} && source deploy_script_all_ref.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year 2018 --append "_ref"

compare-webpage-2018-mc:
  <<: *make-ntuples-102
  <<: *compare-webpage
  dependencies:
    - build-new
    - cmsrun-2018-mc-new
    - cmsrun-2018-mc-ref

eosdeploy-2018-mc:
  <<: *make-ntuples-102
  <<: *eos-deploy
  dependencies:
    - compare-webpage-2018-mc

#-------------------------------------------------------------------------------

cmsrun-2017v2-data-new:
  # Run 2017v2 data on PR
  <<: *make-ntuples-102
  <<: *cmsrun-new
  script:
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year 2017v2 --isData --append "_new"

cmsrun-2017v2-data-ref:
  # Run 2017v2 data on reference
  <<: *make-ntuples-102
  <<: *cmsrun-ref
  script:
    - cd ${TESTDIR} && source deploy_script_all_ref.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year 2017v2 --isData --append "_ref"

compare-webpage-2017v2-data:
  <<: *make-ntuples-102
  <<: *compare-webpage
  dependencies:
    - build-new
    - cmsrun-2017v2-data-new
    - cmsrun-2017v2-data-ref

eosdeploy-2017v2-data:
  <<: *make-ntuples-102
  <<: *eos-deploy
  dependencies:
    - compare-webpage-2017v2-data

#-------------------------------------------------------------------------------

cmsrun-2017v2-mc-new:
  # Run 2017v2 mc on PR
  <<: *make-ntuples-102
  <<: *cmsrun-new
  script:
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year 2017v2 --append "_new"

cmsrun-2017v2-mc-ref:
  # Run 2017v2 mc on reference
  <<: *make-ntuples-102
  <<: *cmsrun-ref
  script:
    - cd ${TESTDIR} && source deploy_script_all_ref.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year 2017v2 --append "_ref"

compare-webpage-2017v2-mc:
  <<: *make-ntuples-102
  <<: *compare-webpage
  dependencies:
    - build-new
    - cmsrun-2017v2-mc-new
    - cmsrun-2017v2-mc-ref

eosdeploy-2017v2-mc:
  <<: *make-ntuples-102
  <<: *eos-deploy
  dependencies:
    - compare-webpage-2017v2-mc

#-------------------------------------------------------------------------------
# NB no 2017v1 data jobs as no one uses that dataset

cmsrun-2017v1-mc-new:
  # Run 2017v1 mc on PR
  <<: *make-ntuples-102
  <<: *cmsrun-new
  script:
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year 2017v1 --append "_new"

cmsrun-2017v1-mc-ref:
  # Run 2017v1 mc on reference
  <<: *make-ntuples-102
  <<: *cmsrun-ref
  script:
    - cd ${TESTDIR} && source deploy_script_all_ref.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year 2017v1 --append "_ref"

compare-webpage-2017v1-mc:
  <<: *make-ntuples-102
  <<: *compare-webpage
  dependencies:
    - build-new
    - cmsrun-2017v1-mc-new
    - cmsrun-2017v1-mc-ref

eosdeploy-2017v1-mc:
  <<: *make-ntuples-102
  <<: *eos-deploy
  dependencies:
    - compare-webpage-2017v1-mc

#-------------------------------------------------------------------------------

cmsrun-2016v3-data-new:
  # Run 2016v3 data on PR
  <<: *make-ntuples-102
  <<: *cmsrun-new
  script:
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year 2016v3 --isData --append "_new"

cmsrun-2016v3-data-ref:
  # Run 2016v3 data on reference
  <<: *make-ntuples-102
  <<: *cmsrun-ref
  script:
    - cd ${TESTDIR} && source deploy_script_all_ref.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year 2016v3 --isData --append "_ref"

compare-webpage-2016v3-data:
  <<: *make-ntuples-102
  <<: *compare-webpage
  dependencies:
    - build-new
    - cmsrun-2016v3-data-new
    - cmsrun-2016v3-data-ref

eosdeploy-2016v3-data:
  <<: *make-ntuples-102
  <<: *eos-deploy
  dependencies:
    - compare-webpage-2016v3-data

#-------------------------------------------------------------------------------

cmsrun-2016v3-mc-new:
  # Run 2016v3 mc on PR
  <<: *make-ntuples-102
  <<: *cmsrun-new
  script:
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year 2016v3 --append "_new"

cmsrun-2016v3-mc-ref:
  # Run 2016v3 mc on reference
  <<: *make-ntuples-102
  <<: *cmsrun-ref
  script:
    - cd ${TESTDIR} && source deploy_script_all_ref.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year 2016v3 --append "_ref"

compare-webpage-2016v3-mc:
  <<: *make-ntuples-102
  <<: *compare-webpage
  dependencies:
    - build-new
    - cmsrun-2016v3-mc-new
    - cmsrun-2016v3-mc-ref

eosdeploy-2016v3-mc:
  <<: *make-ntuples-102
  <<: *eos-deploy
  dependencies:
    - compare-webpage-2016v3-mc

#-------------------------------------------------------------------------------

cmsrun-2016v2-data-new:
  # Run 2016v2 data on PR
  <<: *make-ntuples-102
  <<: *cmsrun-new
  script:
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year 2016v2 --isData --append "_new"

cmsrun-2016v2-data-ref:
  # Run 2016v2 data on reference
  <<: *make-ntuples-102
  <<: *cmsrun-ref
  script:
    - cd ${TESTDIR} && source deploy_script_all_ref.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year 2016v2 --isData --append "_ref"

compare-webpage-2016v2-data:
  <<: *make-ntuples-102
  <<: *compare-webpage
  dependencies:
    - build-new
    - cmsrun-2016v2-data-new
    - cmsrun-2016v2-data-ref

eosdeploy-2016v2-data:
  <<: *make-ntuples-102
  <<: *eos-deploy
  dependencies:
    - compare-webpage-2016v2-data

#-------------------------------------------------------------------------------

cmsrun-2016v2-mc-new:
  # Run 2016v2 mc on PR
  <<: *make-ntuples-102
  <<: *cmsrun-new
  script:
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year 2016v2 --append "_new"

cmsrun-2016v2-mc-ref:
  # Run 2016v2 mc on reference
  <<: *make-ntuples-102
  <<: *cmsrun-ref
  script:
    - cd ${TESTDIR} && source deploy_script_all_ref.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year 2016v2 --append "_ref"

compare-webpage-2016v2-mc:
  <<: *make-ntuples-102
  <<: *compare-webpage
  dependencies:
    - build-new
    - cmsrun-2016v2-mc-new
    - cmsrun-2016v2-mc-ref

eosdeploy-2016v2-mc:
  <<: *make-ntuples-102
  <<: *eos-deploy
  dependencies:
    - compare-webpage-2016v2-mc

#-------------------------------------------------------------------------------
# Ultra-Legacy jobs
#-------------------------------------------------------------------------------
cmsrun-UL18-data-new:
  # Run UL18 data on PR
  <<: *make-ntuples-106
  <<: *cmsrun-new
  script:
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year UL18 --isData --append "_new"

cmsrun-UL18-data-ref:
  # Run UL18 data on reference
  <<: *make-ntuples-106
  <<: *cmsrun-ref
  script:
    - cd ${TESTDIR} && source deploy_script_all_ref.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year UL18 --isData --append "_ref"

compare-webpage-UL18-data:
  <<: *make-ntuples-106
  <<: *compare-webpage
  dependencies:
    - build-new
    - cmsrun-UL18-data-new
    - cmsrun-UL18-data-ref

eosdeploy-UL18-data:
  <<: *make-ntuples-106
  <<: *eos-deploy
  dependencies:
    - compare-webpage-UL18-data

#-------------------------------------------------------------------------------

cmsrun-UL18-mc-new:
  # Run UL18 mc on PR
  <<: *make-ntuples-106
  <<: *cmsrun-new
  script:
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year UL18 --append "_new"

cmsrun-UL18-mc-ref:
  # Run UL18 mc on reference
  <<: *make-ntuples-106
  <<: *cmsrun-ref
  script:
    - cd ${TESTDIR} && source deploy_script_all_ref.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year UL18 --append "_ref"

compare-webpage-UL18-mc:
  <<: *make-ntuples-106
  <<: *compare-webpage
  dependencies:
    - build-new
    - cmsrun-UL18-mc-new
    - cmsrun-UL18-mc-ref

eosdeploy-UL18-mc:
  <<: *make-ntuples-106
  <<: *eos-deploy
  dependencies:
    - compare-webpage-UL18-mc

#-------------------------------------------------------------------------------

cmsrun-UL17-data-new:
  # Run UL17 data on PR
  <<: *make-ntuples-106
  <<: *cmsrun-new
  script:
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year UL17 --isData --append "_new"

cmsrun-UL17-data-ref:
  # Run UL17 data on reference
  <<: *make-ntuples-106
  <<: *cmsrun-ref
  script:
    - cd ${TESTDIR} && source deploy_script_all_ref.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year UL17 --isData --append "_ref"

compare-webpage-UL17-data:
  <<: *make-ntuples-106
  <<: *compare-webpage
  dependencies:
    - build-new
    - cmsrun-UL17-data-new
    - cmsrun-UL17-data-ref

eosdeploy-UL17-data:
  <<: *make-ntuples-106
  <<: *eos-deploy
  dependencies:
    - compare-webpage-UL17-data

#-------------------------------------------------------------------------------

cmsrun-UL17-mc-new:
  # Run UL17 mc on PR
  <<: *make-ntuples-106
  <<: *cmsrun-new
  script:
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year UL17 --append "_new"

cmsrun-UL17-mc-ref:
  # Run UL17 mc on reference
  <<: *make-ntuples-106
  <<: *cmsrun-ref
  script:
    - cd ${TESTDIR} && source deploy_script_all_ref.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year UL17 --append "_ref"

compare-webpage-UL17-mc:
  <<: *make-ntuples-106
  <<: *compare-webpage
  dependencies:
    - build-new
    - cmsrun-UL17-mc-new
    - cmsrun-UL17-mc-ref

eosdeploy-UL17-mc:
  <<: *make-ntuples-106
  <<: *eos-deploy
  dependencies:
    - compare-webpage-UL17-mc

#-------------------------------------------------------------------------------

cmsrun-UL16preVFP-data-new:
  # Run UL16preVFP data on PR
  <<: *make-ntuples-106
  <<: *cmsrun-new
  script:
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year UL16preVFP --isData --append "_new"

cmsrun-UL16preVFP-data-ref:
  # Run UL16preVFP data on reference
  <<: *make-ntuples-106
  <<: *cmsrun-ref
  script:
    - cd ${TESTDIR} && source deploy_script_all_ref.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year UL16preVFP --isData --append "_ref"

compare-webpage-UL16preVFP-data:
  <<: *make-ntuples-106
  <<: *compare-webpage
  dependencies:
    - build-new
    - cmsrun-UL16preVFP-data-new
    - cmsrun-UL16preVFP-data-ref

eosdeploy-UL16preVFP-data:
  <<: *make-ntuples-106
  <<: *eos-deploy
  dependencies:
    - compare-webpage-UL16preVFP-data

#-------------------------------------------------------------------------------

cmsrun-UL16preVFP-mc-new:
  # Run UL16preVFP mc on PR
  <<: *make-ntuples-106
  <<: *cmsrun-new
  script:
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year UL16preVFP --append "_new"

cmsrun-UL16preVFP-mc-ref:
  # Run UL16preVFP mc on reference
  <<: *make-ntuples-106
  <<: *cmsrun-ref
  script:
    - cd ${TESTDIR} && source deploy_script_all_ref.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year UL16preVFP --append "_ref"

compare-webpage-UL16preVFP-mc:
  <<: *make-ntuples-106
  <<: *compare-webpage
  dependencies:
    - build-new
    - cmsrun-UL16preVFP-mc-new
    - cmsrun-UL16preVFP-mc-ref

eosdeploy-UL16preVFP-mc:
  <<: *make-ntuples-106
  <<: *eos-deploy
  dependencies:
    - compare-webpage-UL16preVFP-mc

#-------------------------------------------------------------------------------

cmsrun-UL16postVFP-data-new:
  # Run UL16postVFP data on PR
  <<: *make-ntuples-106
  <<: *cmsrun-new
  script:
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year UL16postVFP --isData --append "_new"

cmsrun-UL16postVFP-data-ref:
  # Run UL16postVFP data on reference
  <<: *make-ntuples-106
  <<: *cmsrun-ref
  script:
    - cd ${TESTDIR} && source deploy_script_all_ref.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year UL16postVFP --isData --append "_ref"

compare-webpage-UL16postVFP-data:
  <<: *make-ntuples-106
  <<: *compare-webpage
  dependencies:
    - build-new
    - cmsrun-UL16postVFP-data-new
    - cmsrun-UL16postVFP-data-ref

eosdeploy-UL16postVFP-data:
  <<: *make-ntuples-106
  <<: *eos-deploy
  dependencies:
    - compare-webpage-UL16postVFP-data

#-------------------------------------------------------------------------------

cmsrun-UL16postVFP-mc-new:
  # Run UL16postVFP mc on PR
  <<: *make-ntuples-106
  <<: *cmsrun-new
  script:
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year UL16postVFP --append "_new"

cmsrun-UL16postVFP-mc-ref:
  # Run UL16postVFP mc on reference
  <<: *make-ntuples-106
  <<: *cmsrun-ref
  script:
    - cd ${TESTDIR} && source deploy_script_all_ref.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt  # to get awkward0, etc
    - python ${SCRIPTDIR}/cmsrun_jobs.py --year UL16postVFP --append "_ref"

compare-webpage-UL16postVFP-mc:
  <<: *make-ntuples-106
  <<: *compare-webpage
  dependencies:
    - build-new
    - cmsrun-UL16postVFP-mc-new
    - cmsrun-UL16postVFP-mc-ref

eosdeploy-UL16postVFP-mc:
  <<: *make-ntuples-106
  <<: *eos-deploy
  dependencies:
    - compare-webpage-UL16postVFP-mc

# -----------------------------------------------------------------------------
# REVIEW
# -----------------------------------------------------------------------------
review-ntuples:
  stage: review
  only:
    variables:
      - $MAKENTUPLES == "1"
  dependencies:
    - build-new
    # 102X jobs:
    - cmsrun-2018-data-new
    - cmsrun-2018-data-ref
    - compare-webpage-2018-data
    - cmsrun-2018-mc-new
    - cmsrun-2018-mc-ref
    - compare-webpage-2018-mc
    - cmsrun-2017v2-data-new
    - cmsrun-2017v2-data-ref
    - compare-webpage-2017v2-data
    - cmsrun-2017v2-mc-new
    - cmsrun-2017v2-mc-ref
    - compare-webpage-2017v2-mc
    - cmsrun-2017v1-mc-new
    - cmsrun-2017v1-mc-ref
    - compare-webpage-2017v1-mc
    - cmsrun-2016v3-data-new
    - cmsrun-2016v3-data-ref
    - compare-webpage-2016v3-data
    - cmsrun-2016v3-mc-new
    - cmsrun-2016v3-mc-ref
    - compare-webpage-2016v3-mc
    - cmsrun-2016v2-data-new
    - cmsrun-2016v2-data-ref
    - compare-webpage-2016v2-data
    - cmsrun-2016v2-mc-new
    - cmsrun-2016v2-mc-ref
    - compare-webpage-2016v2-mc
    # 106X ultra-legacy jobs:
    - cmsrun-UL18-data-new
    - cmsrun-UL18-data-ref
    - compare-webpage-UL18-data
    - eosdeploy-UL18-data
    - cmsrun-UL18-mc-new
    - cmsrun-UL18-mc-ref
    - compare-webpage-UL18-mc
    - eosdeploy-UL18-mc
    - cmsrun-UL17-data-new
    - cmsrun-UL17-data-ref
    - compare-webpage-UL17-data
    - eosdeploy-UL17-data
    - cmsrun-UL17-mc-new
    - cmsrun-UL17-mc-ref
    - compare-webpage-UL17-mc
    - eosdeploy-UL17-mc
    - cmsrun-UL16preVFP-data-new
    - cmsrun-UL16preVFP-data-ref
    - compare-webpage-UL16preVFP-data
    - eosdeploy-UL16preVFP-data
    - cmsrun-UL16preVFP-mc-new
    - cmsrun-UL16preVFP-mc-ref
    - compare-webpage-UL16preVFP-mc
    - eosdeploy-UL16preVFP-mc
    - cmsrun-UL16postVFP-data-new
    - cmsrun-UL16postVFP-data-ref
    - compare-webpage-UL16postVFP-data
    - eosdeploy-UL16postVFP-data
    - cmsrun-UL16postVFP-mc-new
    - cmsrun-UL16postVFP-mc-ref
    - compare-webpage-UL16postVFP-mc
    - eosdeploy-UL16postVFP-mc
  artifacts:
    paths:
      - ${CI_PROJECT_DIR}/review-success
    expire_in: 2 days
  script:
    # Deploy our release to get a newer version of python etc, check packages
    - cd ${TESTDIR} && source deploy_script_all_${CI_COMMIT_REF_NAME}.sh && cd ${TESTDIR}
    - pip install --user -r ${CI_PROJECT_DIR}/requirements_local.txt
    # Make the markdown tables to go into PR text
    - source ${SCRIPTDIR}/makeTimingTable.sh
    - source ${SCRIPTDIR}/makeSizeTable.sh
    - source ${SCRIPTDIR}/makeNtupleComparisonTableAll.sh
    # do PR review with all the necessary inputs
    - ls ${TESTDIR}
    - python ${SCRIPTDIR}/doPRReview.py --timing ${TESTDIR}/timing_report.md --size ${TESTDIR}/size_report.md --plots ${TESTDIR}/ntuple_report.md
    # for later pass/fail check
    - touch ${CI_PROJECT_DIR}/review-success
  after_script:
    # always run, even if one line in script fails
    - if [[ $MAKENTUPLES == "1" ]] && [[ ! -f ${CI_PROJECT_DIR}/review-success ]]; then source ${SCRIPTDIR}/notify_github.sh "failed" "Ntuples / plotting / webpage / review failed, please check logs"; fi


# -----------------------------------------------------------------------------
# FINISHING UP
# -----------------------------------------------------------------------------
final:
  stage: end
  except:
    refs:
      - master
  when: always
  dependencies:
    - build-new
    - review-ntuples
  script:
    # issue final pass/fail messages, based on prescence of files made in earlier stages
    # bit hacky, but no other way - when: on_failure always gets triggered, even if rerun with all stages passing
    - if [[ ! -f ${CI_PROJECT_DIR}/build-success ]]; then source ${SCRIPTDIR}/notify_github.sh "failed" "Compilations and/or tests failed, please check logs"; fi
    # do final success message only when no review, since review does notify_github.sh for us
    - if [[ $MAKENTUPLES == "0" ]]; then source ${SCRIPTDIR}/notify_github.sh "passed" "Compilations & tests successful"; fi


stages:
  - getit
  - build
  - cmsrun
  - compare
  - deploy
  - review
  - end