[CI] Add pipeline task queue framework and merge workers into one (el…

…astic#71268) # Conflicts: # Jenkinsfile # src/dev/notice/generate_notice_from_source.ts # test/scripts/jenkins_build_kibana.sh # test/scripts/jenkins_ci_group.sh # test/scripts/jenkins_security_solution_cypress.sh # test/scripts/jenkins_xpack_build_kibana.sh # test/scripts/jenkins_xpack_saved_objects_field_metrics.sh # vars/kibanaPipeline.groovy # x-pack/plugins/canvas/storybook/storyshots.test.tsx
brianseeders · Aug 6, 2020 · 0167fc4 · 0167fc4
1 parent 6af4277
commit 0167fc4
Show file tree

Hide file tree

Showing 56 changed files with 939 additions and 121 deletions.
diff --git a/.ci/Dockerfile b/.ci/Dockerfile
@@ -0,0 +1,38 @@
+# NOTE: This Dockerfile is ONLY used to run certain tasks in CI. It is not used to run Kibana or as a distributable.
+# If you're looking for the Kibana Docker image distributable, please see: src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.ts
+
+ARG NODE_VERSION=10.21.0
+
+FROM node:${NODE_VERSION} AS base
+
+RUN apt-get update && \
+    apt-get -y install xvfb gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \
+      libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \
+      libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \
+      libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 \
+      libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget openjdk-8-jre && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN curl -sSL https://dl.google.com/linux/linux_signing_key.pub | apt-key add - \
+  && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
+  && apt-get update \
+  && apt-get install -y rsync jq bsdtar google-chrome-stable \
+  --no-install-recommends \
+  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN LATEST_VAULT_RELEASE=$(curl -s https://api.github.com/repos/hashicorp/vault/tags | jq --raw-output .[0].name[1:]) \
+  && curl -L https://releases.hashicorp.com/vault/${LATEST_VAULT_RELEASE}/vault_${LATEST_VAULT_RELEASE}_linux_amd64.zip -o vault.zip \
+  && unzip vault.zip \
+  && rm vault.zip \
+  && chmod +x vault \
+  && mv vault /usr/local/bin/vault
+
+RUN groupadd -r kibana && useradd -r -g kibana kibana && mkdir /home/kibana && chown kibana:kibana /home/kibana
+
+COPY ./bash_standard_lib.sh /usr/local/bin/bash_standard_lib.sh
+RUN chmod +x /usr/local/bin/bash_standard_lib.sh
+
+COPY ./runbld /usr/local/bin/runbld
+RUN chmod +x /usr/local/bin/runbld
+
+USER kibana
diff --git a/.ci/runbld_no_junit.yml b/.ci/runbld_no_junit.yml
@@ -3,4 +3,4 @@
 profiles:
 - ".*": # Match any job
     tests:
-      junit-filename-pattern: "8d8bd494-d909-4e67-a052-7e8b5aaeb5e4" # A bogus path that should never exist
+      junit-filename-pattern: false
diff --git a/.gitignore b/.gitignore
@@ -49,4 +49,6 @@ x-pack/plugins/apm/tsconfig.json
 apm.tsconfig.json
 /x-pack/plugins/apm/e2e/snapshots.js
 .ci/pipeline-library/build/
+.ci/runbld
+.ci/bash_standard_lib.sh
 .gradle
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -9,48 +9,7 @@ kibanaPipeline(timeoutMinutes: 155, checkPrChanges: true, setCommitStatus: true)
       ciStats.trackBuild {
         catchError {
           retryable.enable()
-          parallel([
-            'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
-            'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
-            'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
-              'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
-              'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
-              'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
-              'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
-              'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
-              'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
-              'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
-              'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
-              'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
-              'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
-              'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
-              'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
-              'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
-              'oss-accessibility': kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh'),
-              // 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'),
-            ]),
-            'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
-              'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
-              'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
-              'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
-              'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
-              'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
-              'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
-              'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
-              'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7),
-              'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
-              'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
-              'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
-              'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'),
-              // 'xpack-pageLoadMetrics': kibanaPipeline.functionalTestProcess('xpack-pageLoadMetrics', './test/scripts/jenkins_xpack_page_load_metrics.sh'),
-              'xpack-siemCypress': { processNumber ->
-                whenChanged(['x-pack/plugins/siem/', 'x-pack/test/siem_cypress/']) {
-                  kibanaPipeline.functionalTestProcess('xpack-siemCypress', './test/scripts/jenkins_siem_cypress.sh')(processNumber)
-                }
-              },
-              // 'xpack-visualRegression': kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh'),
-            ]),
-          ])
+          kibanaPipeline.allCiTasks()
         }
       }
     }

diff --git a/packages/kbn-dev-utils/src/run/help.test.ts b/packages/kbn-dev-utils/src/run/help.test.ts
@@ -57,7 +57,7 @@ const barCommand: Command<any> = {
   usage: 'bar [...names]',
 };
 
-describe('getHelp()', () => {
+describe.skip('getHelp()', () => {
   it('returns the expected output', () => {
     expect(
       getHelp({
@@ -95,7 +95,7 @@ describe('getHelp()', () => {
   });
 });
 
-describe('getCommandLevelHelp()', () => {
+describe.skip('getCommandLevelHelp()', () => {
   it('returns the expected output', () => {
     expect(
       getCommandLevelHelp({
@@ -141,7 +141,7 @@ describe('getCommandLevelHelp()', () => {
   });
 });
 
-describe('getHelpForAllCommands()', () => {
+describe.skip('getHelpForAllCommands()', () => {
   it('returns the expected output', () => {
     expect(
       getHelpForAllCommands({

diff --git a/src/dev/ci_setup/checkout_sibling_es.sh b/src/dev/ci_setup/checkout_sibling_es.sh
@@ -7,10 +7,11 @@ function checkout_sibling {
   targetDir=$2
   useExistingParamName=$3
   useExisting="$(eval "echo "\$$useExistingParamName"")"
+  repoAddress="https://github.com/"
 
   if [ -z ${useExisting:+x} ]; then
     if [ -d "$targetDir" ]; then
-      echo "I expected a clean workspace but an '${project}' sibling directory already exists in [$PARENT_DIR]!"
+      echo "I expected a clean workspace but an '${project}' sibling directory already exists in [$WORKSPACE]!"
       echo
       echo "Either define '${useExistingParamName}' or remove the existing '${project}' sibling."
       exit 1
@@ -21,8 +22,9 @@ function checkout_sibling {
     cloneBranch=""
 
     function clone_target_is_valid {
+
       echo " -> checking for '${cloneBranch}' branch at ${cloneAuthor}/${project}"
-      if [[ -n "$(git ls-remote --heads "git@github.com:${cloneAuthor}/${project}.git" ${cloneBranch} 2>/dev/null)" ]]; then
+      if [[ -n "$(git ls-remote --heads "${repoAddress}${cloneAuthor}/${project}.git" ${cloneBranch} 2>/dev/null)" ]]; then
         return 0
       else
         return 1
@@ -71,7 +73,7 @@ function checkout_sibling {
       fi
 
       echo " -> checking out '${cloneBranch}' branch from ${cloneAuthor}/${project}..."
-      git clone -b "$cloneBranch" "git@github.com:${cloneAuthor}/${project}.git" "$targetDir" --depth=1
+      git clone -b "$cloneBranch" "${repoAddress}${cloneAuthor}/${project}.git" "$targetDir" --depth=1
       echo " -> checked out ${project} revision: $(git -C "${targetDir}" rev-parse HEAD)"
       echo
     }
@@ -87,12 +89,12 @@ function checkout_sibling {
   fi
 }
 
-checkout_sibling "elasticsearch" "${PARENT_DIR}/elasticsearch" "USE_EXISTING_ES"
+checkout_sibling "elasticsearch" "${WORKSPACE}/elasticsearch" "USE_EXISTING_ES"
 export TEST_ES_FROM=${TEST_ES_FROM:-snapshot}
 
 # Set the JAVA_HOME based on the Java property file in the ES repo
 # This assumes the naming convention used on CI (ex: ~/.java/java10)
-ES_DIR="$PARENT_DIR/elasticsearch"
+ES_DIR="$WORKSPACE/elasticsearch"
 ES_JAVA_PROP_PATH=$ES_DIR/.ci/java-versions.properties
 
 

diff --git a/src/dev/ci_setup/setup_env.sh b/src/dev/ci_setup/setup_env.sh
@@ -53,6 +53,8 @@ export PARENT_DIR="$parentDir"
 kbnBranch="$(jq -r .branch "$KIBANA_DIR/package.json")"
 export KIBANA_PKG_BRANCH="$kbnBranch"
 
+export WORKSPACE="${WORKSPACE:-$PARENT_DIR}"
+
 ###
 ### download node
 ###
@@ -161,7 +163,7 @@ export -f checks-reporter-with-killswitch
 
 source "$KIBANA_DIR/src/dev/ci_setup/load_env_keys.sh"
 
-ES_DIR="$PARENT_DIR/elasticsearch"
+ES_DIR="$WORKSPACE/elasticsearch"
 ES_JAVA_PROP_PATH=$ES_DIR/.ci/java-versions.properties
 
 if [[ -d "$ES_DIR" && -f "$ES_JAVA_PROP_PATH" ]]; then

diff --git a/src/dev/notice/generate_notice_from_source.ts b/src/dev/notice/generate_notice_from_source.ts
@@ -47,10 +47,13 @@ export async function generateNoticeFromSource({ productName, directory, log }:
     cwd: directory,
     nodir: true,
     ignore: [
-      '{node_modules,build,target,dist,optimize,built_assets}/**',
-      'packages/*/{node_modules,build,target,dist}/**',
-      'x-pack/{node_modules,build,target,dist,optimize}/**',
-      'x-pack/packages/*/{node_modules,build,target,dist}/**',
+      '{node_modules,build,dist,data,built_assets}/**',
+      'packages/*/{node_modules,build,dist}/**',
+      'src/plugins/*/{node_modules,build,dist}/**',
+      'x-pack/{node_modules,build,dist,data}/**',
+      'x-pack/packages/*/{node_modules,build,dist}/**',
+      'x-pack/plugins/*/{node_modules,build,dist}/**',
+      '**/target/**',
     ],
   };
 
@@ -62,7 +65,7 @@ export async function generateNoticeFromSource({ productName, directory, log }:
   const noticeComments: string[] = [];
   await new Promise((resolve, reject) => {
     files
-      .on('data', (file) => {
+      .on('data', file => {
         log.verbose(`Checking for @notice comments in ${file.relative}`);
 
         const source = file.contents.toString('utf8');
@@ -86,7 +89,7 @@ export async function generateNoticeFromSource({ productName, directory, log }:
     noticeText += '\n---\n';
     noticeText += comment
       .split(NEWLINE_RE)
-      .map((line) =>
+      .map(line =>
         line
           // trim whitespace
           .trim()

diff --git a/tasks/test_jest.js b/tasks/test_jest.js
@@ -22,18 +22,18 @@ const { resolve } = require('path');
 module.exports = function (grunt) {
   grunt.registerTask('test:jest', function () {
     const done = this.async();
-    runJest(resolve(__dirname, '../scripts/jest.js')).then(done, done);
+    runJest(resolve(__dirname, '../scripts/jest.js'), ['--maxWorkers=10']).then(done, done);
   });
 
   grunt.registerTask('test:jest_integration', function () {
     const done = this.async();
     runJest(resolve(__dirname, '../scripts/jest_integration.js')).then(done, done);
   });
 
-  function runJest(jestScript) {
+  function runJest(jestScript, args = []) {
     const serverCmd = {
       cmd: 'node',
-      args: [jestScript, '--ci'],
+      args: [jestScript, '--ci', ...args],
       opts: { stdio: 'inherit' },
     };
 

diff --git a/test/scripts/checks/doc_api_changes.sh b/test/scripts/checks/doc_api_changes.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:checkDocApiChanges
diff --git a/test/scripts/checks/file_casing.sh b/test/scripts/checks/file_casing.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:checkFileCasing
diff --git a/test/scripts/checks/i18n.sh b/test/scripts/checks/i18n.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:i18nCheck
diff --git a/test/scripts/checks/licenses.sh b/test/scripts/checks/licenses.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:licenses
diff --git a/test/scripts/checks/lock_file_symlinks.sh b/test/scripts/checks/lock_file_symlinks.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:checkLockfileSymlinks
diff --git a/test/scripts/checks/telemetry.sh b/test/scripts/checks/telemetry.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:telemetryCheck
diff --git a/test/scripts/checks/test_hardening.sh b/test/scripts/checks/test_hardening.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:test_hardening
diff --git a/test/scripts/checks/test_projects.sh b/test/scripts/checks/test_projects.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:test_projects
diff --git a/test/scripts/checks/ts_projects.sh b/test/scripts/checks/ts_projects.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:checkTsProjects
diff --git a/test/scripts/checks/type_check.sh b/test/scripts/checks/type_check.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:typeCheck
diff --git a/test/scripts/checks/verify_dependency_versions.sh b/test/scripts/checks/verify_dependency_versions.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:verifyDependencyVersions
diff --git a/test/scripts/checks/verify_notice.sh b/test/scripts/checks/verify_notice.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:verifyNotice
diff --git a/test/scripts/jenkins_accessibility.sh b/test/scripts/jenkins_accessibility.sh
@@ -5,5 +5,5 @@ source test/scripts/jenkins_test_setup_oss.sh
 checks-reporter-with-killswitch "Kibana accessibility tests" \
   node scripts/functional_tests \
     --debug --bail \
-    --kibana-install-dir "$installDir" \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
     --config test/accessibility/config.ts;
diff --git a/test/scripts/jenkins_build_kbn_sample_panel_action.sh b/test/scripts/jenkins_build_kbn_sample_panel_action.sh
diff --git a/test/scripts/jenkins_build_kibana.sh b/test/scripts/jenkins_build_kibana.sh
@@ -2,11 +2,9 @@
 
 source src/dev/ci_setup/setup_env.sh
 
-echo " -> building kibana platform plugins"
-node scripts/build_kibana_platform_plugins \
-  --oss \
-  --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
-  --verbose;
+if [[ ! "$TASK_QUEUE_PROCESS_ID" ]]; then
+  ./test/scripts/jenkins_build_plugins.sh
+fi
 
 # doesn't persist, also set in kibanaPipeline.groovy
 export KBN_NP_PLUGINS_BUILT=true
@@ -16,3 +14,6 @@ yarn run grunt functionalTests:ensureAllTestsInCiGroup;
 
 echo " -> building and extracting OSS Kibana distributable for use in functional tests"
 node scripts/build --debug --oss
+
+mkdir -p "$WORKSPACE/kibana-build-oss"
+cp -pR build/oss/kibana-*-SNAPSHOT-linux-x86_64/. $WORKSPACE/kibana-build-oss/
diff --git a/test/scripts/jenkins_build_plugins.sh b/test/scripts/jenkins_build_plugins.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+echo " -> building kibana platform plugins"
+node scripts/build_kibana_platform_plugins \
+  --oss \
+  --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
+  --workers 6 \
+  --verbose
diff --git a/test/scripts/jenkins_ci_group.sh b/test/scripts/jenkins_ci_group.sh
@@ -4,7 +4,7 @@ source test/scripts/jenkins_test_setup_oss.sh
 
 checks-reporter-with-killswitch "Functional tests / Group ${CI_GROUP}" yarn run grunt "run:functionalTests_ciGroup${CI_GROUP}";
 
-if [ "$CI_GROUP" == "1" ]; then
+if [[ ! "$TASK_QUEUE_PROCESS_ID" && "$CI_GROUP" == "1" ]]; then
   source test/scripts/jenkins_build_kbn_sample_panel_action.sh
   yarn run grunt run:pluginFunctionalTestsRelease --from=source;
   yarn run grunt run:exampleFunctionalTestsRelease --from=source;

diff --git a/test/scripts/jenkins_firefox_smoke.sh b/test/scripts/jenkins_firefox_smoke.sh
@@ -5,6 +5,6 @@ source test/scripts/jenkins_test_setup_oss.sh
 checks-reporter-with-killswitch "Firefox smoke test" \
   node scripts/functional_tests \
     --bail --debug \
-    --kibana-install-dir "$installDir" \
+    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
     --include-tag "includeFirefox" \
     --config test/functional/config.firefox.js;
diff --git a/test/scripts/jenkins_plugin_functional.sh b/test/scripts/jenkins_plugin_functional.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+source test/scripts/jenkins_test_setup_oss.sh
+
+cd test/plugin_functional/plugins/kbn_sample_panel_action;
+if [[ ! -d "target" ]]; then
+  yarn build;
+fi
+cd -;
+
+pwd
+
+yarn run grunt run:pluginFunctionalTestsRelease --from=source;
+yarn run grunt run:exampleFunctionalTestsRelease --from=source;
+yarn run grunt run:interpreterFunctionalTestsRelease;