[CI] disable aws cloud testing by default (#23186) (#23387)

* Disable by default, and added the labels/parameters/comments configuration to be customised on a PR basis if required
* Created a weekly job to run AWS cloud testing for master and 7.x branches

.ci/jobs/beats-schedule-weekly.yml

@@ -0,0 +1,27 @@
+---
+- job:
+    name: beats-schedule-weekly
+    display-name: Jobs scheduled weekly (Sunday)
+    description: Jobs scheduled weekly (Sunday)
+    view: Beats
+    project-type: pipeline
+    parameters:
+      - string:
+          name: branch_specifier
+          default: master
+          description: the Git branch specifier to build
+    pipeline-scm:
+      script-path: .ci/schedule-weekly.groovy
+      scm:
+        - git:
+            url: git@github.com:elastic/beats.git
+            refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/beats.git
+            branches:
+              - $branch_specifier
+    triggers:
+      - timed: 'H H(1-4) * * 0'

.ci/schedule-weekly.groovy

@@ -0,0 +1,33 @@
+@Library('apm@current') _
+
+pipeline {
+  agent none
+  environment {
+    NOTIFY_TO = credentials('notify-to')
+    PIPELINE_LOG_LEVEL = 'INFO'
+  }
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+  }
+  triggers {
+    cron('H H(1-4) * * 0')
+  }
+  stages {
+    stage('Nighly beats builds') {
+      steps {
+        build(quietPeriod: 0, job: 'Beats/beats/master', parameters: [booleanParam(name: 'awsCloudTests', value: true)])
+        build(quietPeriod: 1000, job: 'Beats/beats/7.x', parameters: [booleanParam(name: 'awsCloudTests', value: true)])
+      }
+    }
+  }
+  post {
+    cleanup {
+      notifyBuildResult(prComment: false)
+    }
+  }
+}

Jenkinsfile

@@ -37,7 +37,7 @@ pipeline {
   }
   parameters {
     booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.')
-    booleanParam(name: 'awsCloudTests', defaultValue: true, description: 'Run AWS cloud integration tests.')
+    booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.')
     string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.')
     booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.')
     booleanParam(name: 'armTest', defaultValue: false, description: 'Allow ARM stages.')

x-pack/metricbeat/Jenkinsfile.yml

@@ -18,11 +18,20 @@ stages:
           make -C x-pack/metricbeat check;
           make -C x-pack/metricbeat update;
           make check-no-changes;
-    build:
+    unitTest:
+        mage: "mage build unitTest"
+    cloud:
         cloud: "mage build test"
         withModule: true       ## run the ITs only if the changeset affects a specific module.
         dirs:                  ## run the cloud tests for the given modules.
             - "x-pack/metricbeat/module/aws"
+        when:                  ## Override the top-level when.
+            parameters:
+                - "awsCloudTests"
+            comments:
+                - "/test x-pack/metricbeat for aws cloud"
+            labels:
+                - "aws"
     macos:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.