Merge branch 'master' into update-indices

.buildkite/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "kibana-buildkite",
+  "version": "1.0.0",
+  "private": true,
+  "dependencies": {
+    "kibana-buildkite-library": "elastic/kibana-buildkite-library"
+  }
+}

.buildkite/pipelines/es_snapshots/build.yml

@@ -0,0 +1,5 @@
+steps:
+  - command: .buildkite/scripts/steps/es_snapshots/build.sh
+    label: Build ES Snapshot
+    agents:
+      queue: c2-8

.buildkite/pipelines/es_snapshots/promote.yml

@@ -0,0 +1,12 @@
+steps:
+  - block: 'Promote'
+    prompt: "Enter the details for the snapshot you'd like to promote"
+    if: "build.env('ES_SNAPSHOT_MANIFEST') == null"
+    # Later, this could be a dropdown dynamically filled with recent builds
+    fields:
+      - text: 'ES_SNAPSHOT_MANIFEST'
+        key: 'ES_SNAPSHOT_MANIFEST'
+        hint: 'URL pointing to the manifest to promote'
+        required: true
+  - label: Promote Snapshot
+    command: .buildkite/scripts/steps/es_snapshots/promote.sh

.buildkite/pipelines/es_snapshots/verify.yml

@@ -0,0 +1,102 @@
+env:
+  IGNORE_SHIP_CI_STATS_ERROR: 'true'
+steps:
+  - block: 'Verify'
+    prompt: "Enter the details for the snapshot you'd like to verify"
+    if: "build.env('ES_SNAPSHOT_MANIFEST') == null"
+    # Later, this could be a dropdown dynamically filled with recent builds
+    fields:
+      - text: 'ES_SNAPSHOT_MANIFEST'
+        key: 'ES_SNAPSHOT_MANIFEST'
+        hint: 'URL pointing to the manifest to promote'
+        required: true
+
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+
+  - wait
+
+  - command: .buildkite/scripts/steps/build_kibana.sh
+    label: Build Kibana Distribution and Plugins
+    agents:
+      queue: c2-8
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+
+  - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
+    label: 'Default CI Group'
+    parallelism: 13
+    agents:
+      queue: ci-group-6
+    artifact_paths: target/junit/**/*.xml
+    depends_on: build
+    key: default-cigroup
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
+    label: 'Docker CI Group'
+    agents:
+      queue: ci-group-6
+    artifact_paths: target/junit/**/*.xml
+    depends_on: build
+    key: default-cigroup-docker
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
+    label: 'OSS CI Group'
+    parallelism: 12
+    agents:
+      queue: ci-group-4d
+    artifact_paths: target/junit/**/*.xml
+    depends_on: build
+    key: oss-cigroup
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/test/jest_integration.sh
+    label: 'Jest Integration Tests'
+    agents:
+      queue: jest
+    artifact_paths: target/junit/**/*.xml
+    key: jest-integration
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/test/api_integration.sh
+    label: 'API Integration Tests'
+    agents:
+      queue: jest
+    artifact_paths: target/junit/**/*.xml
+    key: api-integration
+
+  - command: .buildkite/scripts/steps/es_snapshots/trigger_promote.sh
+    label: Trigger promotion
+    depends_on:
+      - default-cigroup
+      - default-cigroup-docker
+      - oss-cigroup
+      - jest-integration
+      - api-integration
+
+  - wait: ~
+    continue_on_failure: true
+
+  - plugins:
+      - junit-annotate#v1.9.0:
+          artifacts: target/junit/**/*.xml
+
+  - wait: ~
+    continue_on_failure: true
+
+  - command: .buildkite/scripts/lifecycle/post_build.sh
+    label: Post-Build

.buildkite/pipelines/pull_request.yml

@@ -0,0 +1,17 @@
+env:
+  GITHUB_COMMIT_STATUS_ENABLED: 'true'
+  GITHUB_COMMIT_STATUS_CONTEXT: 'buildkite/kibana-pull-request'
+steps:
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+
+  - wait
+
+  - command: echo 'Hello World'
+    label: Test
+
+  - wait: ~
+    continue_on_failure: true
+
+  - command: .buildkite/scripts/lifecycle/post_build.sh
+    label: Post-Build

.buildkite/pipelines/update_demo_env.yml

@@ -0,0 +1,8 @@
+steps:
+  - command: .buildkite/scripts/steps/demo_env/es_and_init.sh
+    label: Initialize Environment and Deploy ES
+
+  - command: .buildkite/scripts/steps/demo_env/kibana.sh
+    label: Build and Deploy Kibana
+    agents:
+      queue: c2-8

.buildkite/scripts/bootstrap.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 source .buildkite/scripts/common/util.sh
 
 echo "--- yarn install and bootstrap"
-yarn kbn bootstrap --verbose
+yarn kbn bootstrap
 
 ###
 ### upload ts-refs-cache artifacts as quickly as possible so they are available for download

.buildkite/scripts/build_kibana_plugins.sh

@@ -18,8 +18,7 @@ node scripts/build_kibana_platform_plugins \
   --scan-dir "$XPACK_DIR/test/licensing_plugin/plugins" \
   --scan-dir "$XPACK_DIR/test/usage_collection/plugins" \
   --scan-dir "$XPACK_DIR/test/security_functional/fixtures/common" \
-  --scan-dir "$XPACK_DIR/examples" \
-  --verbose
+  --scan-dir "$XPACK_DIR/examples"
 
 echo "--- Archive built plugins"
 shopt -s globstar

.buildkite/scripts/common/env.sh

@@ -70,3 +70,4 @@ export TEST_ES_URL="http://elastic:changeme@localhost:6102"
 export TEST_ES_TRANSPORT_PORT=6301-6309
 export TEST_CORS_SERVER_PORT=6106
 export ALERTING_PROXY_PORT=6105
+export TEST_PROXY_SERVER_PORT=6107

.buildkite/scripts/download_build_artifacts.sh

@@ -7,8 +7,8 @@ if [[ ! -d "$KIBANA_BUILD_LOCATION/bin" ]]; then
 
   cd "$WORKSPACE"
 
-  buildkite-agent artifact download kibana-default.tar.gz .
-  buildkite-agent artifact download kibana-default-plugins.tar.gz .
+  buildkite-agent artifact download kibana-default.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  buildkite-agent artifact download kibana-default-plugins.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 
   mkdir -p "$KIBANA_BUILD_LOCATION"
   tar -xzf kibana-default.tar.gz -C "$KIBANA_BUILD_LOCATION" --strip=1

.buildkite/scripts/lifecycle/build_status.js

@@ -0,0 +1,17 @@
+const { BuildkiteClient } = require('kibana-buildkite-library');
+
+(async () => {
+  try {
+    const client = new BuildkiteClient();
+    const status = await client.getCurrentBuildStatus();
+    console.log(status.success ? 'true' : 'false');
+    process.exit(0);
+  } catch (ex) {
+    console.error('Buildkite API Error', ex.message);
+    if (ex.response) {
+      console.error('HTTP Error Response Status', ex.response.status);
+      console.error('HTTP Error Response Body', ex.response.data);
+    }
+    process.exit(1);
+  }
+})();

.buildkite/scripts/lifecycle/ci_stats_complete.js

@@ -1,17 +1,14 @@
-const ciStats = require('./ci_stats');
-
-// TODO - this is okay for now but should really be replaced with an API call, especially once retries are enabled
-const BUILD_STATUS = process.env.BUILD_FAILED === 'true' ? 'FAILURE' : 'SUCCESS';
+const { CiStats } = require('kibana-buildkite-library');
 
 (async () => {
   try {
-    if (process.env.CI_STATS_BUILD_ID) {
-      await ciStats.post(`/v1/build/_complete?id=${process.env.CI_STATS_BUILD_ID}`, {
-        result: BUILD_STATUS,
-      });
-    }
+    await CiStats.onComplete();
   } catch (ex) {
-    console.error(ex);
+    console.error('CI Stats Error', ex.message);
+    if (ex.response) {
+      console.error('HTTP Error Response Status', ex.response.status);
+      console.error('HTTP Error Response Body', ex.response.data);
+    }
     process.exit(1);
   }
 })();

.buildkite/scripts/lifecycle/ci_stats_start.js

@@ -1,30 +1,14 @@
-const { execSync } = require('child_process');
-const ciStats = require('./ci_stats');
+const { CiStats } = require('kibana-buildkite-library');
 
 (async () => {
   try {
-    const build = await ciStats.post('/v1/build', {
-      jenkinsJobName: process.env.BUILDKITE_PIPELINE_NAME,
-      jenkinsJobId: process.env.BUILDKITE_BUILD_ID,
-      jenkinsUrl: process.env.BUILDKITE_BUILD_URL,
-      prId: process.env.GITHUB_PR_NUMBER || null,
-    });
-
-    execSync(`buildkite-agent meta-data set ci_stats_build_id "${build.id}"`);
-
-    // TODO Will need to set MERGE_BASE for PRs
-
-    await ciStats.post(`/v1/git_info?buildId=${build.id}`, {
-      branch: process.env.BUILDKITE_BRANCH.replace(/^(refs\/heads\/|origin\/)/, ''),
-      commit: process.env.BUILDKITE_COMMIT,
-      targetBranch:
-        process.env.GITHUB_PR_TARGET_BRANCH ||
-        process.env.BUILDKITE_PULL_REQUEST_BASE_BRANCH ||
-        null,
-      mergeBase: process.env.GITHUB_PR_MERGE_BASE || null, // TODO confirm GITHUB_PR_MERGE_BASE or switch to final var
-    });
+    await CiStats.onStart();
   } catch (ex) {
-    console.error(ex);
+    console.error('CI Stats Error', ex.message);
+    if (ex.response) {
+      console.error('HTTP Error Response Status', ex.response.status);
+      console.error('HTTP Error Response Body', ex.response.data);
+    }
     process.exit(1);
   }
 })();

.buildkite/scripts/lifecycle/commit_status_complete.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 
 if [[ "${GITHUB_COMMIT_STATUS_ENABLED:-}" == "true" ]]; then
   COMMIT_STATUS=success
-  if [[ "${BUILD_FAILED:-}"  == "true" ]]; then
+  if [[ "${BUILD_SUCCESSFUL:-}"  != "true" ]]; then
     COMMIT_STATUS=failure
   fi
 

.buildkite/scripts/lifecycle/post_build.sh

@@ -2,8 +2,8 @@
 
 set -euo pipefail
 
-BUILD_FAILED=$(buildkite-agent meta-data get build_failed --default "false")
-export BUILD_FAILED
+BUILD_SUCCESSFUL=$(node "$(dirname "${0}")/build_status.js")
+export BUILD_SUCCESSFUL
 
 "$(dirname "${0}")/commit_status_complete.sh"
 

.buildkite/scripts/lifecycle/pre_command.sh

@@ -2,6 +2,13 @@
 
 set -euo pipefail
 
+cd '.buildkite'
+yarn install
+cd -
+
+BUILDKITE_TOKEN="$(vault read -field=buildkite_token_all_jobs secret/kibana-issues/dev/buildkite-ci)"
+export BUILDKITE_TOKEN
+
 # Set up a custom ES Snapshot Manifest if one has been specified for this build
 {
   ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST --default '')}
@@ -65,3 +72,8 @@ if [[ "${SKIP_CI_SETUP:-}" != "true" ]]; then
     source .buildkite/scripts/common/setup_bazel.sh
   fi
 fi
+
+PIPELINE_PRE_COMMAND=${PIPELINE_PRE_COMMAND:-".buildkite/scripts/lifecycle/pipelines/$BUILDKITE_PIPELINE_SLUG/pre_command.sh"}
+if [[ -f "$PIPELINE_PRE_COMMAND" ]]; then
+  source "$PIPELINE_PRE_COMMAND"
+fi

.buildkite/scripts/steps/build_kibana.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+.buildkite/scripts/build_kibana.sh
+.buildkite/scripts/build_kibana_plugins.sh
+.buildkite/scripts/post_build_kibana.sh