Merge branch 'main' into issues/128695

.buildkite/pipelines/artifacts.yml

@@ -4,3 +4,29 @@ steps:
     agents:
       queue: c2-16
     timeout_in_minutes: 60
+
+  - wait
+
+  - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh
+    label: Package testing for deb
+    agents:
+      queue: n2-4-virt
+    timeout_in_minutes: 20
+
+  - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh
+    label: Package testing for rpm
+    agents:
+      queue: n2-4-virt
+    timeout_in_minutes: 20
+
+  - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh
+    label: Package testing for docker
+    agents:
+      queue: n2-4-virt
+    timeout_in_minutes: 20
+
+  - command: .buildkite/scripts/steps/artifacts/docker_context.sh
+    label: 'Docker Build Context'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 20

.buildkite/pipelines/pull_request/base.yml

@@ -56,7 +56,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_accessibility.sh
     label: 'OSS Accessibility Tests'
     agents:
-      queue: ci-group-4d
+      queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -67,7 +67,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_accessibility.sh
     label: 'Default Accessibility Tests'
     agents:
-      queue: n2-4
+      queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -78,7 +78,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_firefox.sh
     label: 'OSS Firefox Tests'
     agents:
-      queue: ci-group-4d
+      queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -89,7 +89,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_firefox.sh
     label: 'Default Firefox Tests'
     agents:
-      queue: n2-4
+      queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -100,7 +100,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_misc.sh
     label: 'OSS Misc Functional Tests'
     agents:
-      queue: n2-4
+      queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -111,7 +111,7 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_saved_object_field_metrics.sh
     label: 'Saved Object Field Metrics'
     agents:
-      queue: n2-4
+      queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -138,7 +138,7 @@ steps:
   - command: .buildkite/scripts/steps/test/api_integration.sh
     label: 'API Integration Tests'
     agents:
-      queue: n2-2
+      queue: n2-2-spot
     timeout_in_minutes: 120
     key: api-integration
 
@@ -173,7 +173,7 @@ steps:
   - command: .buildkite/scripts/steps/build_api_docs.sh
     label: 'Build API Docs'
     agents:
-      queue: n2-4
+      queue: n2-4-spot
     key: build_api_docs
     timeout_in_minutes: 60
 

.buildkite/scripts/common/util.sh

@@ -47,6 +47,11 @@ check_for_changed_files() {
 
       git commit -m "$NEW_COMMIT_MESSAGE"
       git push
+
+      # After the git push, the new commit will trigger a new build within a few seconds and this build should get cancelled
+      # So, let's just sleep to give the build time to cancel itself without an error
+      # If it doesn't get cancelled for some reason, then exit with an error, because we don't want this build to be green (we just don't want it to generate an error either)
+      sleep 300
       exit 1
     else
       echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n"

.buildkite/scripts/lifecycle/annotate_test_failures.js

@@ -6,7 +6,6 @@
  * Side Public License, v 1.
  */
 
-// eslint-disable-next-line import/no-unresolved
 const { TestFailures } = require('kibana-buildkite-library');
 
 (async () => {

.buildkite/scripts/lifecycle/build_status.js

@@ -6,7 +6,6 @@
  * Side Public License, v 1.
  */
 
-// eslint-disable-next-line import/no-unresolved
 const { BuildkiteClient } = require('kibana-buildkite-library');
 
 (async () => {

.buildkite/scripts/lifecycle/ci_stats_complete.js

@@ -6,7 +6,6 @@
  * Side Public License, v 1.
  */
 
-// eslint-disable-next-line import/no-unresolved
 const { CiStats } = require('kibana-buildkite-library');
 
 (async () => {

.buildkite/scripts/lifecycle/ci_stats_ready.js

@@ -6,7 +6,6 @@
  * Side Public License, v 1.
  */
 
-// eslint-disable-next-line import/no-unresolved
 const { CiStats } = require('kibana-buildkite-library');
 
 (async () => {

.buildkite/scripts/lifecycle/ci_stats_start.js

@@ -6,7 +6,6 @@
  * Side Public License, v 1.
  */
 
-// eslint-disable-next-line import/no-unresolved
 const { CiStats } = require('kibana-buildkite-library');
 
 (async () => {

.buildkite/scripts/lifecycle/print_agent_links.js

@@ -6,7 +6,6 @@
  * Side Public License, v 1.
  */
 
-// eslint-disable-next-line import/no-unresolved
 const { BuildkiteClient } = require('kibana-buildkite-library');
 
 (async () => {

.buildkite/scripts/pipelines/pull_request/pipeline.js

@@ -8,7 +8,6 @@
 
 const execSync = require('child_process').execSync;
 const fs = require('fs');
-// eslint-disable-next-line import/no-unresolved
 const { areChangesSkippable, doAnyChangesMatch } = require('kibana-buildkite-library');
 
 const SKIPPABLE_PATHS = [

.buildkite/scripts/steps/artifacts/build.sh

@@ -17,3 +17,31 @@ node scripts/build "$RELEASE_ARG" --all-platforms --debug --docker-cross-compile
 
 echo "--- Build dependencies report"
 node scripts/licenses_csv_report "--csv=target/dependencies-$VERSION.csv"
+
+# Release verification
+if [[ "${RELEASE_BUILD:-}" == "true" ]]; then
+  echo "--- Build and push Kibana Cloud Distribution"
+  # This doesn't meet the requirements for a release image, implementation TBD
+  # Beats artifacts will need to match a specific commit sha that matches other stack iamges
+  # For now this is a placeholder step that will allow us to run automated Cloud tests
+  # against a best guess approximation of a release image
+  echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+  trap 'docker logout docker.elastic.co' EXIT
+
+  node scripts/build \
+    "$RELEASE_ARG" \
+    --skip-initialize \
+    --skip-generic-folders \
+    --skip-platform-folders \
+    --skip-archives \
+    --docker-images \
+    --docker-tag-qualifier="$GIT_COMMIT" \
+    --docker-push \
+    --skip-docker-ubi \
+    --skip-docker-ubuntu \
+    --skip-docker-contexts
+fi
+
+cd target
+buildkite-agent artifact upload "*"
+cd -

.buildkite/scripts/steps/artifacts/docker_context.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+
+if [[ "${RELEASE_BUILD:-}" == "true" ]]; then
+  VERSION="$(jq -r '.version' package.json)"
+  RELEASE_ARG="--release"
+else
+  VERSION="$(jq -r '.version' package.json)-SNAPSHOT"
+  RELEASE_ARG=""
+fi
+
+echo "--- Create contexts"
+mkdir -p target
+node scripts/build "$RELEASE_ARG" --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact
+
+echo "--- Setup default context"
+DOCKER_BUILD_FOLDER=$(mktemp -d)
+
+tar -xf target/kibana-[0-9]*-docker-build-context.tar.gz -C "$DOCKER_BUILD_FOLDER"
+cd $DOCKER_BUILD_FOLDER
+
+buildkite-agent artifact download "kibana-$VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+
+echo "--- Build context"
+docker build .

.buildkite/scripts/steps/es_snapshots/build.sh

@@ -69,6 +69,7 @@ echo "--- Build Elasticsearch"
   :distribution:archives:darwin-aarch64-tar:assemble \
   :distribution:archives:darwin-tar:assemble \
   :distribution:docker:docker-export:assemble \
+  :distribution:docker:cloud-docker-export:assemble \
   :distribution:archives:linux-aarch64-tar:assemble \
   :distribution:archives:linux-tar:assemble \
   :distribution:archives:windows-zip:assemble \
@@ -79,11 +80,26 @@ find distribution -type f \( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elas
 
 ls -alh "$destination"
 
-echo "--- Create docker image archives"
+echo "--- Create docker default image archives"
 docker images "docker.elastic.co/elasticsearch/elasticsearch"
 docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 echo 'docker save docker.elastic.co/elasticsearch/elasticsearch:${0} | gzip > ../es-build/elasticsearch-${0}-docker-image.tar.gz'
 docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 bash -c 'docker save docker.elastic.co/elasticsearch/elasticsearch:${0} | gzip > ../es-build/elasticsearch-${0}-docker-image.tar.gz'
 
+echo "--- Create kibana-ci docker cloud image archives"
+ES_CLOUD_ID=$(docker images "docker.elastic.co/elasticsearch-ci/elasticsearch-cloud" --format "{{.ID}}")
+ES_CLOUD_VERSION=$(docker images "docker.elastic.co/elasticsearch-ci/elasticsearch-cloud" --format "{{.Tag}}")
+KIBANA_ES_CLOUD_VERSION="$ES_CLOUD_VERSION-$ELASTICSEARCH_GIT_COMMIT"
+KIBANA_ES_CLOUD_IMAGE="docker.elastic.co/kibana-ci/elasticsearch-cloud:$KIBANA_ES_CLOUD_VERSION"
+
+docker tag "$ES_CLOUD_ID" "$KIBANA_ES_CLOUD_IMAGE"
+
+echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+trap 'docker logout docker.elastic.co' EXIT
+docker image push "$KIBANA_ES_CLOUD_IMAGE"
+
+export ELASTICSEARCH_CLOUD_IMAGE="$KIBANA_ES_CLOUD_IMAGE"
+export ELASTICSEARCH_CLOUD_IMAGE_CHECKSUM="$(docker images "$KIBANA_ES_CLOUD_IMAGE" --format "{{.Digest}}")"
+
 echo "--- Create checksums for snapshot files"
 cd "$destination"
 find ./* -exec bash -c "shasum -a 512 {} > {}.sha512" \;

.buildkite/scripts/steps/es_snapshots/create_manifest.js

@@ -16,6 +16,8 @@ const { BASE_BUCKET_DAILY } = require('./bucket_config.js');
   const destination = process.argv[2] || __dirname + '/test';
 
   const ES_BRANCH = process.env.ELASTICSEARCH_BRANCH;
+  const ES_CLOUD_IMAGE = process.env.ELASTICSEARCH_CLOUD_IMAGE;
+  const ES_CLOUD_IMAGE_CHECKSUM = process.env.ELASTICSEARCH_CLOUD_IMAGE_CHECKSUM;
   const GIT_COMMIT = process.env.ELASTICSEARCH_GIT_COMMIT;
   const GIT_COMMIT_SHORT = process.env.ELASTICSEARCH_GIT_COMMIT_SHORT;
 
@@ -59,6 +61,17 @@ const { BASE_BUCKET_DAILY } = require('./bucket_config.js');
         };
       });
 
+    if (ES_CLOUD_IMAGE && ES_CLOUD_IMAGE_CHECKSUM) {
+      manifestEntries.push({
+        checksum: ES_CLOUD_IMAGE_CHECKSUM,
+        url: ES_CLOUD_IMAGE,
+        version: VERSION,
+        platform: 'docker',
+        architecture: 'image',
+        license: 'default',
+      });
+    }
+
     const manifest = {
       id: SNAPSHOT_ID,
       bucket: `${BASE_BUCKET_DAILY}/${DESTINATION}`.toString(),

.buildkite/scripts/steps/lint.sh

@@ -9,7 +9,25 @@ source .buildkite/scripts/common/util.sh
 echo '--- Lint: stylelint'
 checks-reporter-with-killswitch "Lint: stylelint" \
   node scripts/stylelint
+echo "stylelint ✅"
+
+# disable "Exit immediately" mode so that we can run eslint, capture it's exit code, and respond appropriately
+# after possibly commiting fixed files to the repo
+set +e;
 
 echo '--- Lint: eslint'
 checks-reporter-with-killswitch "Lint: eslint" \
-  node scripts/eslint --no-cache
+  node scripts/eslint --no-cache --fix
+
+eslint_exit=$?
+
+# re-enable "Exit immediately" mode
+set -e;
+
+check_for_changed_files 'node scripts/eslint --no-cache --fix' true
+
+if [[ "${eslint_exit}" != "0" ]]; then
+  exit 1
+fi
+
+echo "eslint ✅"

.eslintrc.js

@@ -274,12 +274,7 @@ module.exports = {
      * Licence headers
      */
     {
-      files: [
-        '**/*.{js,mjs,ts,tsx}',
-        '!plugins/**/*',
-        '!packages/elastic-datemath/**/*',
-        '!packages/elastic-eslint-config-kibana/**/*',
-      ],
+      files: ['**/*.{js,mjs,ts,tsx}'],
       rules: {
         '@kbn/eslint/require-license-header': [
           'error',
@@ -584,30 +579,6 @@ module.exports = {
       },
     },
 
-    /**
-     * Files that are allowed to import webpack-specific stuff
-     */
-    {
-      files: [
-        '**/public/**/*.js',
-        'src/fixtures/**/*.js', // TODO: this directory needs to be more obviously "public" (or go away)
-      ],
-      settings: {
-        // instructs import/no-extraneous-dependencies to treat certain modules
-        // as core modules, even if they aren't listed in package.json
-        'import/core-modules': ['plugins'],
-
-        'import/resolver': {
-          '@kbn/eslint-import-resolver-kibana': {
-            forceNode: false,
-            rootPackageName: 'kibana',
-            kibanaPath: '.',
-            pluginMap: {},
-          },
-        },
-      },
-    },
-
     /**
      * Single package.json rules, it tells eslint to ignore the child package.json files
      * and look for dependencies declarations in the single and root level package.json
@@ -695,7 +666,6 @@ module.exports = {
     {
       files: [
         '.eslintrc.js',
-        'packages/kbn-eslint-import-resolver-kibana/**/*.js',
         'packages/kbn-eslint-plugin-eslint/**/*',
         'x-pack/gulpfile.js',
         'x-pack/scripts/*.js',
@@ -1572,14 +1542,6 @@ module.exports = {
     {
       files: ['x-pack/plugins/canvas/canvas_plugin_src/**/*.js'],
       globals: { canvas: true, $: true },
-      rules: {
-        'import/no-unresolved': [
-          'error',
-          {
-            ignore: ['!!raw-loader.+.svg$'],
-          },
-        ],
-      },
     },
     {
       files: ['x-pack/plugins/canvas/public/**/*.js'],