Merge branch 'master' into ingest-pipelines/processor-forms-a-d

.ci/Dockerfile

@@ -0,0 +1,38 @@
+# NOTE: This Dockerfile is ONLY used to run certain tasks in CI. It is not used to run Kibana or as a distributable.
+# If you're looking for the Kibana Docker image distributable, please see: src/dev/build/tasks/os_packages/docker_generator/templates/dockerfile.template.ts
+
+ARG NODE_VERSION=10.21.0
+
+FROM node:${NODE_VERSION} AS base
+
+RUN apt-get update && \
+    apt-get -y install xvfb gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \
+      libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \
+      libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \
+      libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 \
+      libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget openjdk-8-jre && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN curl -sSL https://dl.google.com/linux/linux_signing_key.pub | apt-key add - \
+  && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
+  && apt-get update \
+  && apt-get install -y rsync jq bsdtar google-chrome-stable \
+  --no-install-recommends \
+  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN LATEST_VAULT_RELEASE=$(curl -s https://api.github.com/repos/hashicorp/vault/tags | jq --raw-output .[0].name[1:]) \
+  && curl -L https://releases.hashicorp.com/vault/${LATEST_VAULT_RELEASE}/vault_${LATEST_VAULT_RELEASE}_linux_amd64.zip -o vault.zip \
+  && unzip vault.zip \
+  && rm vault.zip \
+  && chmod +x vault \
+  && mv vault /usr/local/bin/vault
+
+RUN groupadd -r kibana && useradd -r -g kibana kibana && mkdir /home/kibana && chown kibana:kibana /home/kibana
+
+COPY ./bash_standard_lib.sh /usr/local/bin/bash_standard_lib.sh
+RUN chmod +x /usr/local/bin/bash_standard_lib.sh
+
+COPY ./runbld /usr/local/bin/runbld
+RUN chmod +x /usr/local/bin/runbld
+
+USER kibana

.ci/Jenkinsfile_baseline_capture

@@ -7,18 +7,22 @@ kibanaPipeline(timeoutMinutes: 120) {
   githubCommitStatus.trackBuild(params.commit, 'kibana-ci-baseline') {
     ciStats.trackBuild {
       catchError {
-        parallel([
-          'oss-visualRegression': {
-            workers.ci(name: 'oss-visualRegression', size: 's-highmem', ramDisk: true) {
-              kibanaPipeline.functionalTestProcess('oss-visualRegression', './test/scripts/jenkins_visual_regression.sh')(1)
-            }
-          },
-          'xpack-visualRegression': {
-            workers.ci(name: 'xpack-visualRegression', size: 's-highmem', ramDisk: true) {
-              kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')(1)
-            }
-          },
-        ])
+        withEnv([
+          'CI_PARALLEL_PROCESS_NUMBER=1'
+        ]) {
+          parallel([
+            'oss-visualRegression': {
+              workers.ci(name: 'oss-visualRegression', size: 's-highmem', ramDisk: true) {
+                kibanaPipeline.functionalTestProcess('oss-visualRegression', './test/scripts/jenkins_visual_regression.sh')()
+              }
+            },
+            'xpack-visualRegression': {
+              workers.ci(name: 'xpack-visualRegression', size: 's-highmem', ramDisk: true) {
+                kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')()
+              }
+            },
+          ])
+        }
       }
 
       kibanaPipeline.sendMail()

.ci/pipeline-library/src/test/KibanaBasePipelineTest.groovy

@@ -78,6 +78,10 @@ class KibanaBasePipelineTest extends BasePipelineTest {
     return helper.callStack.find { it.methodName == name }
   }
 
+  def fnMocks(String name) {
+    helper.callStack.findAll { it.methodName == name }
+  }
+
   void mockFailureBuild() {
     props([
       buildUtils: [

.ci/pipeline-library/src/test/prChanges.groovy

@@ -97,4 +97,17 @@ class PrChangesTest extends KibanaBasePipelineTest {
 
     assertFalse(prChanges.areChangesSkippable())
   }
+
+  @Test
+  void 'areChangesSkippable() with plugin readme changes'() {
+    props([
+      githubPrs: [
+        getChanges: { [
+          [filename: 'src/plugins/foo/README.asciidoc'],
+        ] },
+      ],
+    ])
+
+    assertFalse(prChanges.areChangesSkippable())
+  }
 }

.ci/pipeline-library/src/test/slackNotifications.groovy

@@ -9,6 +9,7 @@ class SlackNotificationsTest extends KibanaBasePipelineTest {
     super.setUp()
 
     helper.registerAllowedMethod('slackSend', [Map.class], null)
+    prop('buildState', loadScript("vars/buildState.groovy"))
     slackNotifications = loadScript('vars/slackNotifications.groovy')
   }
 
@@ -25,13 +26,49 @@ class SlackNotificationsTest extends KibanaBasePipelineTest {
   }
 
   @Test
-  void 'sendFailedBuild() should call slackSend() with message'() {
+  void 'sendFailedBuild() should call slackSend() with an in-progress message'() {
     mockFailureBuild()
 
     slackNotifications.sendFailedBuild()
 
     def args = fnMock('slackSend').args[0]
 
+    def expected = [
+      channel: '#kibana-operations-alerts',
+      username: 'Kibana Operations',
+      iconEmoji: ':jenkins:',
+      color: 'danger',
+      message: ':hourglass_flowing_sand: elastic / kibana # master #1',
+    ]
+
+    expected.each {
+      assertEquals(it.value.toString(), args[it.key].toString())
+    }
+
+    assertEquals(
+      ":hourglass_flowing_sand: *<http://jenkins.localhost:8080/job/elastic+kibana+master/1/|elastic / kibana # master #1>*",
+      args.blocks[0].text.text.toString()
+    )
+
+    assertEquals(
+      "*Failed Steps*\n• <http://jenkins.localhost:8080|Execute test task>",
+      args.blocks[1].text.text.toString()
+    )
+
+    assertEquals(
+      "*Test Failures*\n• <https://localhost/|x-pack/test/functional/apps/fake/test·ts.Fake test &lt;Component&gt; should &amp; pass &amp;>",
+      args.blocks[2].text.text.toString()
+    )
+  }
+
+  @Test
+  void 'sendFailedBuild() should call slackSend() with message'() {
+    mockFailureBuild()
+
+    slackNotifications.sendFailedBuild(isFinal: true)
+
+    def args = fnMock('slackSend').args[0]
+
     def expected = [
       channel: '#kibana-operations-alerts',
       username: 'Kibana Operations',
@@ -59,4 +96,90 @@ class SlackNotificationsTest extends KibanaBasePipelineTest {
       args.blocks[2].text.text.toString()
     )
   }
+
+  @Test
+  void 'sendFailedBuild() should call slackSend() with a backup message when first attempt fails'() {
+    mockFailureBuild()
+    def counter = 0
+    helper.registerAllowedMethod('slackSend', [Map.class], { ++counter > 1 })
+    slackNotifications.sendFailedBuild(isFinal: true)
+
+    def args = fnMocks('slackSend')[1].args[0]
+
+    def expected = [
+      channel: '#kibana-operations-alerts',
+      username: 'Kibana Operations',
+      iconEmoji: ':jenkins:',
+      color: 'danger',
+      message: ':broken_heart: elastic / kibana # master #1',
+    ]
+
+    expected.each {
+      assertEquals(it.value.toString(), args[it.key].toString())
+    }
+
+    assertEquals(
+      ":broken_heart: *<http://jenkins.localhost:8080/job/elastic+kibana+master/1/|elastic / kibana # master #1>*" +
+        "\n\nFirst attempt at sending this notification failed. Please check the build.",
+      args.blocks[0].text.text.toString()
+    )
+  }
+
+  @Test
+  void 'sendFailedBuild() should call slackSend() with a channel id and timestamp on second call'() {
+    mockFailureBuild()
+    helper.registerAllowedMethod('slackSend', [Map.class], { [ channelId: 'CHANNEL_ID', ts: 'TIMESTAMP' ] })
+    slackNotifications.sendFailedBuild(isFinal: false)
+    slackNotifications.sendFailedBuild(isFinal: true)
+
+    def args = fnMocks('slackSend')[1].args[0]
+
+    def expected = [
+      channel: 'CHANNEL_ID',
+      timestamp: 'TIMESTAMP',
+      username: 'Kibana Operations',
+      iconEmoji: ':jenkins:',
+      color: 'danger',
+      message: ':broken_heart: elastic / kibana # master #1',
+    ]
+
+    expected.each {
+      assertEquals(it.value.toString(), args[it.key].toString())
+    }
+  }
+
+  @Test
+  void 'getTestFailures() should truncate list of failures to 10'() {
+    prop('testUtils', [
+      getFailures: {
+        return (1..12).collect {
+          return [
+            url: Mocks.TEST_FAILURE_URL,
+            fullDisplayName: "Failure #${it}",
+          ]
+        }
+      },
+    ])
+
+    def message = (String) slackNotifications.getTestFailures()
+
+    assertTrue("Message ends with truncated indicator", message.endsWith("...and 2 more"))
+    assertTrue("Message contains Failure #10", message.contains("Failure #10"))
+    assertTrue("Message does not contain Failure #11", !message.contains("Failure #11"))
+  }
+
+  @Test
+  void 'shortenMessage() should truncate a long message, but leave parts that fit'() {
+    assertEquals('Hello\nHello\n[...truncated...]', slackNotifications.shortenMessage('Hello\nHello\nthis is a long string', 29))
+  }
+
+  @Test
+  void 'shortenMessage() should not modify a short message'() {
+    assertEquals('Hello world', slackNotifications.shortenMessage('Hello world', 11))
+  }
+
+  @Test
+  void 'shortenMessage() should truncate an entire message with only one part'() {
+    assertEquals('[...truncated...]', slackNotifications.shortenMessage('Hello world this is a really long message', 40))
+  }
 }

.ci/runbld_no_junit.yml

@@ -3,4 +3,4 @@
 profiles:
 - ".*": # Match any job
     tests:
-      junit-filename-pattern: "8d8bd494-d909-4e67-a052-7e8b5aaeb5e4" # A bogus path that should never exist
+      junit-filename-pattern: false

.eslintrc.js

@@ -132,12 +132,6 @@ module.exports = {
         'react-hooks/rules-of-hooks': 'off',
       },
     },
-    {
-      files: ['x-pack/plugins/lens/**/*.{js,mjs,ts,tsx}'],
-      rules: {
-        'react-hooks/exhaustive-deps': 'off',
-      },
-    },
     {
       files: ['x-pack/plugins/ml/**/*.{js,mjs,ts,tsx}'],
       rules: {
@@ -529,6 +523,7 @@ module.exports = {
         'x-pack/test_utils/**/*',
         'x-pack/gulpfile.js',
         'x-pack/plugins/apm/public/utils/testHelpers.js',
+        'x-pack/plugins/canvas/shareable_runtime/postcss.config.js',
       ],
       rules: {
         'import/no-extraneous-dependencies': [
@@ -771,19 +766,22 @@ module.exports = {
     },
 
     /**
-     * APM overrides
+     * APM and Observability overrides
      */
     {
-      files: ['x-pack/plugins/apm/**/*.js'],
+      files: [
+        'x-pack/plugins/apm/**/*.{js,mjs,ts,tsx}',
+        'x-pack/plugins/observability/**/*.{js,mjs,ts,tsx}',
+      ],
       rules: {
-        'no-unused-vars': ['error', { ignoreRestSiblings: true }],
         'no-console': ['warn', { allow: ['error'] }],
-      },
-    },
-    {
-      plugins: ['react-hooks'],
-      files: ['x-pack/plugins/apm/**/*.{ts,tsx}'],
-      rules: {
+        'react/function-component-definition': [
+          'warn',
+          {
+            namedComponents: 'function-declaration',
+            unnamedComponents: 'arrow-function',
+          },
+        ],
         'react-hooks/rules-of-hooks': 'error', // Checks rules of Hooks
         'react-hooks/exhaustive-deps': ['error', { additionalHooks: '^useFetcher$' }],
       },

.github/CODEOWNERS

@@ -7,7 +7,6 @@
 /x-pack/plugins/discover_enhanced/ @elastic/kibana-app
 /x-pack/plugins/lens/ @elastic/kibana-app
 /x-pack/plugins/graph/ @elastic/kibana-app
-/src/legacy/core_plugins/kibana/public/local_application_service/ @elastic/kibana-app
 /src/plugins/dashboard/ @elastic/kibana-app
 /src/plugins/discover/ @elastic/kibana-app
 /src/plugins/input_control_vis/ @elastic/kibana-app

.gitignore

@@ -49,6 +49,8 @@ npm-debug.log*
 .tern-project
 .nyc_output
 .ci/pipeline-library/build/
+.ci/runbld
+.ci/bash_standard_lib.sh
 .gradle
 
 # apm plugin

Jenkinsfile

@@ -4,59 +4,18 @@ library 'kibana-pipeline-library'
 kibanaLibrary.load()
 
 kibanaPipeline(timeoutMinutes: 155, checkPrChanges: true, setCommitStatus: true) {
-  githubPr.withDefaultPrComments {
-    ciStats.trackBuild {
-      catchError {
-        retryable.enable()
-        parallel([
-          'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
-          'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
-          'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
-            'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
-            'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
-            'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
-            'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
-            'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
-            'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
-            'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
-            'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
-            'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
-            'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
-            'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
-            'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
-            'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
-            'oss-accessibility': kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh'),
-            // 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'),
-          ]),
-          'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
-            'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
-            'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
-            'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
-            'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
-            'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
-            'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
-            'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
-            'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7),
-            'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
-            'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
-            'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
-            'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'),
-            'xpack-savedObjectsFieldMetrics': kibanaPipeline.functionalTestProcess('xpack-savedObjectsFieldMetrics', './test/scripts/jenkins_xpack_saved_objects_field_metrics.sh'),
-            'xpack-securitySolutionCypress': { processNumber ->
-              whenChanged(['x-pack/plugins/security_solution/', 'x-pack/test/security_solution_cypress/']) {
-                kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypress', './test/scripts/jenkins_security_solution_cypress.sh')(processNumber)
-              }
-            },
-
-            // 'xpack-visualRegression': kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh'),
-          ]),
-        ])
+  slackNotifications.onFailure(disabled: !params.NOTIFY_ON_FAILURE) {
+    githubPr.withDefaultPrComments {
+      ciStats.trackBuild {
+        catchError {
+          retryable.enable()
+          kibanaPipeline.allCiTasks()
+        }
       }
     }
   }
 
   if (params.NOTIFY_ON_FAILURE) {
-    slackNotifications.onFailure()
     kibanaPipeline.sendMail()
   }
 }