Merge branch 'main' into 164661-discover-panels-redesign

elastic · Sep 5, 2023 · cd4c90c · cd4c90c
2 parents 3ffc0ff + 212a4a0
commit cd4c90c
Show file tree

Hide file tree

Showing 115 changed files with 3,673 additions and 443 deletions.
diff --git a/.buildkite/pipelines/pipeline.kibana-serverless-release.yaml b/.buildkite/pipelines/pipeline.kibana-serverless-release.yaml
@@ -2,8 +2,11 @@ steps:
   - label: ":releasethekraken: Release kibana"
     # https://regex101.com/r/tY52jo/1
     if: build.tag =~ /^deploy@\d+\$/
-    trigger: gpctl-promote
+    trigger: gpctl-promote-with-e2e-tests
     build:
       env:
         SERVICE_COMMIT_HASH: "${BUILDKITE_COMMIT:0:12}"
         REMOTE_SERVICE_CONFIG: https://raw.githubusercontent.com/elastic/serverless-gitops/main/gen/gpctl/kibana/config.yaml
+        SERVICE: kibana-controller
+        NAMESPACE: kibana-ci
+        IMAGE_NAME: kibana-serverless
diff --git a/.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml b/.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml
@@ -1,4 +1,17 @@
+# This pipeline serves as the entry point for your service's quality gates definitions. When
+# properly configured, it will be invoked automatically as part of the automated
+# promotion process once a new version was rolled out in one of the various cloud stages.
+#
+# The updated environment is provided via ENVIRONMENT variable. The seedling
+# step will branch and execute pipeline snippets at the following location:
+# pipeline.tests-qa.yaml
+# pipeline.tests-staging.yaml
+# pipeline.tests-production.yaml
+#
+# Docs: https://docs.elastic.dev/serverless/qualitygates
+
 env:
+  TEAM_CHANNEL: "#kibana-mission-control"
   ENVIRONMENT: ${ENVIRONMENT?}
 
 steps:
@@ -8,3 +21,7 @@ steps:
     command: "make -C /agent run-environment-tests"
     agents:
       image: "docker.elastic.co/ci-agent-images/quality-gate-seedling:0.0.2"
+
+notify:
+  - slack: "${TEAM_CHANNEL?}"
+    if: build.branch == "main" && build.state == "failed"
diff --git a/.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml b/.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml
@@ -1,6 +1,6 @@
-env:
-  TEAM_CHANNEL: "#kibana-serverless-release"
-  ENVIRONMENT: "production"
+# These pipeline steps constitute the quality gate for your service within the production
+# environment. Incorporate any necessary additional logic to validate the service's integrity.
+# A failure in this pipeline build will prevent further progression to the subsequent stage.
 
 steps:
   - label: ":pipeline::fleet::seedling: Trigger Observability Kibana Tests for ${ENVIRONMENT}"
@@ -13,9 +13,17 @@ steps:
     agents:
       image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364"
 
+  - label: ":rocket: Run cp e2e tests"
+    trigger: "ess-k8s-production-e2e-tests"
+    build:
+      message: "${BUILDKITE_MESSAGE}"
+      env:
+        REGION_ID: aws-us-east-1
+        NAME_PREFIX: ci_test_${SERVICE}-promotion_
+
   - wait: ~
 
   - label: ":judge::seedling: Trigger Manual Tests Phase"
     command: "make -C /agent trigger-manual-verification-phase"
     agents:
-      image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.1"
+      image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.2"
diff --git a/.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml b/.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml
@@ -1,6 +1,6 @@
-env:
-  TEAM_CHANNEL: "#kibana-serverless-release"
-  ENVIRONMENT: "qa"
+# These pipeline steps constitute the quality gate for your service within the QA environment.
+# Incorporate any necessary additional logic to validate the service's integrity. A failure in
+# this pipeline build will prevent further progression to the subsequent stage.
 
 steps:
   - label: ":pipeline::kibana::seedling: Trigger Kibana Tests for ${ENVIRONMENT}"
@@ -23,9 +23,17 @@ steps:
     agents:
       image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364"
 
+  - label: ":rocket: Run cp e2e tests"
+    trigger: "ess-k8s-qa-e2e-tests-daily"
+    build:
+      message: "${BUILDKITE_MESSAGE}"
+      env:
+        REGION_ID: aws-eu-west-1
+        NAME_PREFIX: ci_test_kibana-promotion_
+
   - wait: ~
 
   - label: ":judge::seedling: Trigger Manual Tests Phase"
     command: "make -C /agent trigger-manual-verification-phase"
     agents:
-      image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.1"
+      image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.2"
diff --git a/.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml b/.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml
@@ -1,6 +1,6 @@
-env:
-  TEAM_CHANNEL: "#kibana-serverless-release"
-  ENVIRONMENT: "staging"
+# These pipeline steps constitute the quality gate for your service within the staging environment.
+# Incorporate any necessary additional logic to validate the service's integrity. A failure in
+# this pipeline build will prevent further progression to the subsequent stage.
 
 steps:
   - label: ":pipeline::fleet::seedling: Trigger Observability Kibana Tests for ${ENVIRONMENT}"
@@ -13,9 +13,17 @@ steps:
     agents:
       image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364"
 
+  - label: ":rocket: Run cp e2e tests"
+    trigger: "ess-k8s-staging-e2e-tests"
+    build:
+      message: "${BUILDKITE_MESSAGE}"
+      env:
+        REGION_ID: aws-us-east-1
+        NAME_PREFIX: ci_test_kibana-promotion_
+
   - wait: ~
 
   - label: ":judge::seedling: Trigger Manual Tests Phase"
     command: "make -C /agent trigger-manual-verification-phase"
     agents:
-      image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.1"
+      image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.2"
diff --git a/.github/workflows/create-deploy-tag.yml b/.github/workflows/create-deploy-tag.yml
@@ -19,7 +19,7 @@ concurrency:
 jobs:
   create-deploy-tag:
     # Temporary, we need a way to limit this to a GitHub team instead of specific users
-    if: contains('["watson","clintandrewhall","kobelb","lukeelmers","thomasneirynck"]', github.triggering_actor)
+    if: contains('["watson","clintandrewhall","kobelb","lukeelmers","thomasneirynck","jbudz","mistic","delanni","Ikuni17"]', github.triggering_actor)
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -63,7 +63,7 @@ jobs:
                   "type": "section",
                   "text": {
                     "type": "mrkdwn",
-                    "text": "A new <https://github.com/elastic/kibana/commit/${{ env.COMMIT }}|commit> has been promoted to QA 🎉\n\nOnce promotion is complete, please begin any required manual testing.\n\n*Remember:* Promotion to Staging is currently a manual process and will proceed once the build is signed off in QA."
+                    "text": "Promotion of a new <https://github.com/elastic/kibana/commit/${{ env.COMMIT }}|commit> to QA has been initiated 🎉\n\nOnce promotion is complete, please begin any required manual testing.\n\n*Remember:* Promotion to Staging is currently a manual process and will proceed once the build is signed off in QA."
                   }
                 },
                 {
@@ -73,10 +73,6 @@ jobs:
                       "type": "mrkdwn",
                       "text": "*Initiated by:*\n<https://github.com/${{ github.triggering_actor }}|${{ github.triggering_actor }}>"
                     },
-                    {
-                      "type": "mrkdwn",
-                      "text": "*Workflow run:*\n<https://github.com/elastic/kibana/actions/runs/${{ github.run_id }}|${{ github.run_id }}>"
-                    },
                     {
                       "type": "mrkdwn",
                       "text": "*Commit:*\n<https://github.com/elastic/kibana/commit/${{ env.COMMIT }}|${{ env.COMMIT }}>"
@@ -107,7 +103,7 @@ jobs:
                   "type": "section",
                   "text": {
                     "type": "mrkdwn",
-                    "text": "*Useful links:*\n\n • <https://docs.google.com/document/d/1c2LzojDh1wawjeMsKh4D_L2jpVJALhxukkmmL-TUbrs/edit#heading=h.50173f90utwr|Release process playbook>\n • <https://example.com|QA Quality Gate pipeline>"
+                    "text": "*Useful links:*\n\n • <https://docs.google.com/document/d/1c2LzojDh1wawjeMsKh4D_L2jpVJALhxukkmmL-TUbrs/edit#heading=h.50173f90utwr|Release process playbook>\n • <https://github.com/elastic/kibana/actions/runs/${{ github.run_id }}|GitHub Workflow run>\n • <https://buildkite.com/elastic/kibana-tests/builds?branch=main|Quality Gate pipeline>\n • <https://argo-workflows.us-central1.gcp.qa.cld.elstc.co/workflows?label=hash%3D${{ env.COMMIT }}|Argo Workflow>\n • <https://platform-logging.kb.us-central1.gcp.foundit.no/app/dashboards#/view/f710d7d0-00b9-11ee-93d2-8df4bca5a4c9?_g=(refreshInterval:(pause:!t,value:0),time:(from:now-1d,to:now))|GPCTL Logs dashboard>"
                   }
                 },
                 {
@@ -153,7 +149,7 @@ jobs:
                   "type": "section",
                   "text": {
                     "type": "mrkdwn",
-                    "text": "Promotion of <https://github.com/elastic/kibana/commit/${{ env.COMMIT }}|commit> to QA failed ⛔️"
+                    "text": "Creation of deploy tag on <https://github.com/elastic/kibana/commit/${{ env.COMMIT }}|commit> failed ⛔️"
                   }
                 },
                 {
@@ -163,10 +159,6 @@ jobs:
                       "type": "mrkdwn",
                       "text": "*Initiated by:*\n<https://github.com/${{ github.triggering_actor }}|${{ github.triggering_actor }}>"
                     },
-                    {
-                      "type": "mrkdwn",
-                      "text": "*Workflow run:*\n<https://github.com/elastic/kibana/actions/runs/${{ github.run_id }}|${{ github.run_id }}>"
-                    },
                     {
                       "type": "mrkdwn",
                       "text": "*Commit:*\n<https://github.com/elastic/kibana/commit/${{ env.COMMIT }}|${{ env.COMMIT }}>"
@@ -177,7 +169,7 @@ jobs:
                   "type": "section",
                   "text": {
                     "type": "mrkdwn",
-                    "text": "*Useful links:*\n\n • <https://docs.google.com/document/d/1c2LzojDh1wawjeMsKh4D_L2jpVJALhxukkmmL-TUbrs/edit#heading=h.50173f90utwr|Release process playbook>"
+                    "text": "*Useful links:*\n\n • <https://docs.google.com/document/d/1c2LzojDh1wawjeMsKh4D_L2jpVJALhxukkmmL-TUbrs/edit#heading=h.50173f90utwr|Release process playbook>\n • <https://github.com/elastic/kibana/actions/runs/${{ github.run_id }}|GitHub Workflow run>"
                   }
                 }
               ]

diff --git a/docs/management/connectors/action-types/opsgenie.asciidoc b/docs/management/connectors/action-types/opsgenie.asciidoc
@@ -28,34 +28,6 @@ URL:: The Opsgenie URL. For example, https://api.opsgenie.com or https://api.eu.
 NOTE: If you are using the <<action-settings, `xpack.actions.allowedHosts`>> setting, make sure the hostname is added to the allowed hosts.
 API Key::   The Opsgenie API authentication key for HTTP Basic authentication. For more details about generating Opsgenie API keys, refer to https://support.atlassian.com/opsgenie/docs/create-a-default-api-integration/[Opsgenie documentation].
 
-[float]
-[[preconfigured-opsgenie-configuration]]
-=== Create preconfigured connectors
-
-If you are running {kib} on-prem, you can define connectors by
-adding `xpack.actions.preconfigured` settings to your `kibana.yml` file.
-For example:
-
-[source,text]
---
-xpack.actions.preconfigured:
-  my-opsgenie:
-    name: preconfigured-opsgenie-connector-type
-    actionTypeId: .opsgenie
-    config:
-      apiUrl: https://api.opsgenie.com
-    secrets:
-      apiKey: apikey
---
-
-Config defines information for the connector type.
-
-`apiUrl`:: A string that corresponds to *URL*.
-
-Secrets defines sensitive information for the connector type.
-
-`apiKey`:: A string that corresponds to *API Key*.
-
 [float]
 [[opsgenie-action-configuration]]
 === Test connectors

diff --git a/docs/management/connectors/pre-configured-connectors.asciidoc b/docs/management/connectors/pre-configured-connectors.asciidoc
@@ -83,9 +83,30 @@ configuration.
 [float]
 === Examples
 
+* <<preconfigured-opsgenie-configuration>>
 * <<preconfigured-server-log-configuration>>
 * <<preconfigured-webhook-configuration>>
 
+[float]
+[[preconfigured-opsgenie-configuration]]
+==== {opsgenie} connectors
+
+The following example creates an <<opsgenie-action-type,{opsgenie} connector>>:
+
+[source,text]
+--
+xpack.actions.preconfigured:
+  my-opsgenie:
+    name: preconfigured-opsgenie-connector-type
+    actionTypeId: .opsgenie
+    config:
+      apiUrl: https://api.opsgenie.com <1>
+    secrets:
+      apiKey: apikey <2>
+--
+<1> The {opsgenie} URL.
+<2> The {opsgenie} API authentication key for HTTP basic authentication.
+
 [float]
 [[preconfigured-server-log-configuration]]
 ==== Server log connectors

diff --git a/docs/settings/alert-action-settings.asciidoc b/docs/settings/alert-action-settings.asciidoc
@@ -252,9 +252,31 @@ Specifies configuration details that are specific to the type of preconfigured c
 The type of preconfigured connector.
 For example: `.email`, `.index`, `.opsgenie`, `.server-log`, `.resilient`, `.slack`, and `.webhook`.
 
+`xpack.actions.preconfigured.<connector-id>.config.apiUrl`::
+A configuration URL that varies by connector:
++
+--
+* For an <<opsgenie-action-type,{opsgenie} connector>>, specifies the {opsgenie} URL. For example, `https://api.opsgenie.com` or `https://api.eu.opsgenie.com`.
+
+NOTE: If you are using the `xpack.actions.allowedHosts` setting, make sure the hostname in the URL is added to the allowed hosts.
+--
+
 `xpack.actions.preconfigured.<connector-id>.name`::
 The name of the preconfigured connector.
 
+`xpack.actions.preconfigured.<connector-id>.secrets`::
+Sensitive configuration details, such as username, password, and keys, which are specific to the connector type.
++
+TIP: Sensitive properties, such as passwords, should be stored in the <<creating-keystore,{kib} keystore>>.
+
+`xpack.actions.preconfigured.<connector-id>.secrets.apikey`::
+An API key secret that varies by connector:
++
+--
+* For an <<opsgenie-action-type,{opsgenie} connector>>, specifies the {opsgenie} API authentication key for HTTP basic authentication.
+--
+
+
 [float]
 [[alert-settings]]
 === Alerting settings

diff --git a/package.json b/package.json
@@ -841,6 +841,7 @@
     "core-js": "^3.31.0",
     "cronstrue": "^1.51.0",
     "css-box-model": "^1.2.1",
+    "css.escape": "^1.5.1",
     "cuid": "^2.1.8",
     "cytoscape": "^3.10.0",
     "cytoscape-dagre": "^2.2.2",

diff --git a/packages/kbn-search-api-panels/types.ts b/packages/kbn-search-api-panels/types.ts
@@ -24,6 +24,8 @@ export interface LanguageDefinitionSnippetArguments {
   apiKey: string;
   indexName?: string;
   cloudId?: string;
+  ingestPipeline?: string;
+  extraIngestDocumentValues?: Record<string, boolean>;
 }
 
 type CodeSnippet = string | ((args: LanguageDefinitionSnippetArguments) => string);

diff --git a/packages/kbn-text-based-editor/index.ts b/packages/kbn-text-based-editor/index.ts
@@ -7,6 +7,7 @@
  */
 
 export type { TextBasedLanguagesEditorProps } from './src/text_based_languages_editor';
+export { fetchFieldsFromESQL } from './src/fetch_fields_from_esql';
 import { TextBasedLanguagesEditor } from './src/text_based_languages_editor';
 
 // React.lazy support

diff --git a/packages/kbn-text-based-editor/src/editor_footer.tsx b/packages/kbn-text-based-editor/src/editor_footer.tsx
@@ -155,6 +155,7 @@ interface EditorFooterProps {
   detectTimestamp: boolean;
   onErrorClick: (error: MonacoError) => void;
   refreshErrors: () => void;
+  hideRunQueryText?: boolean;
 }
 
 export const EditorFooter = memo(function EditorFooter({
@@ -165,6 +166,7 @@ export const EditorFooter = memo(function EditorFooter({
   detectTimestamp,
   onErrorClick,
   refreshErrors,
+  hideRunQueryText,
 }: EditorFooterProps) {
   const [isPopoverOpen, setIsPopoverOpen] = useState(false);
   return (
@@ -235,27 +237,29 @@ export const EditorFooter = memo(function EditorFooter({
           </EuiFlexItem>
         </EuiFlexGroup>
       </EuiFlexItem>
-      <EuiFlexItem grow={false}>
-        <EuiFlexGroup gutterSize="xs" responsive={false} alignItems="center">
-          <EuiFlexItem grow={false}>
-            <EuiText size="xs" color="subdued">
-              <p>
-                {i18n.translate('textBasedEditor.query.textBasedLanguagesEditor.runQuery', {
-                  defaultMessage: 'Run query',
-                })}
-              </p>
-            </EuiText>
-          </EuiFlexItem>
-          <EuiFlexItem grow={false}>
-            <EuiCode
-              transparentBackground
-              css={css`
-                font-size: 12px;
-              `}
-            >{`${COMMAND_KEY} + Enter`}</EuiCode>
-          </EuiFlexItem>
-        </EuiFlexGroup>
-      </EuiFlexItem>
+      {!hideRunQueryText && (
+        <EuiFlexItem grow={false}>
+          <EuiFlexGroup gutterSize="xs" responsive={false} alignItems="center">
+            <EuiFlexItem grow={false}>
+              <EuiText size="xs" color="subdued" data-test-subj="TextBasedLangEditor-run-query">
+                <p>
+                  {i18n.translate('textBasedEditor.query.textBasedLanguagesEditor.runQuery', {
+                    defaultMessage: 'Run query',
+                  })}
+                </p>
+              </EuiText>
+            </EuiFlexItem>
+            <EuiFlexItem grow={false}>
+              <EuiCode
+                transparentBackground
+                css={css`
+                  font-size: 12px;
+                `}
+              >{`${COMMAND_KEY} + Enter`}</EuiCode>
+            </EuiFlexItem>
+          </EuiFlexGroup>
+        </EuiFlexItem>
+      )}
     </EuiFlexGroup>
   );
 });