Merge branch 'main' into asset-criticality-alert-enrichment

.buildkite/disabled_jest_configs.json

@@ -1,3 +1 @@
-[
- "x-pack/plugins/index_management/jest.config.js"
-]
+[]

.buildkite/ftr_configs.yml

@@ -341,6 +341,7 @@ enabled:
  - x-pack/test/observability_onboarding_api_integration/basic/config.ts
  - x-pack/test/observability_onboarding_api_integration/cloud/config.ts
  - x-pack/test/observability_ai_assistant_api_integration/enterprise/config.ts
+ - x-pack/test/observability_ai_assistant_functional/enterprise/config.ts
  - x-pack/test/plugin_api_integration/config.ts
  - x-pack/test/plugin_functional/config.ts
  - x-pack/test/reporting_api_integration/reporting_and_security.config.ts

.buildkite/pipelines/on_merge.yml

@@ -271,30 +271,6 @@ steps:
  - exit_status: '*'
  limit: 1
 
- - command: .buildkite/scripts/steps/functional/defend_workflows.sh
- label: 'Defend Workflows Cypress Tests'
- agents:
- queue: n2-4-virt
- depends_on: build
- timeout_in_minutes: 60
- parallelism: 10
- retry:
- automatic:
- - exit_status: '*'
- limit: 1
-
- - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
- label: 'Defend Workflows Cypress Tests on Serverless'
- agents:
- queue: n2-4-virt
- depends_on: build
- timeout_in_minutes: 60
- parallelism: 6
- retry:
- automatic:
- - exit_status: '*'
- limit: 1
-
  - command: .buildkite/scripts/steps/functional/threat_intelligence.sh
  label: 'Threat Intelligence Cypress Tests'
  agents:

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -61,3 +61,27 @@ steps:
  limit: 3
  - exit_status: '*'
  limit: 1
+
+ - command: .buildkite/scripts/steps/functional/defend_workflows.sh
+ label: 'Defend Workflows Cypress Tests'
+ agents:
+ queue: n2-4-virt
+ depends_on: build
+ timeout_in_minutes: 60
+ parallelism: 10
+ retry:
+ automatic:
+ - exit_status: '*'
+ limit: 1
+
+ - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
+ label: 'Defend Workflows Cypress Tests on Serverless'
+ agents:
+ queue: n2-4-virt
+ depends_on: build
+ timeout_in_minutes: 60
+ parallelism: 6
+ retry:
+ automatic:
+ - exit_status: '*'
+ limit: 1

.buildkite/pipelines/pull_request/base.yml

@@ -249,30 +249,6 @@ steps:
  - exit_status: '*'
  limit: 1
 
- - command: .buildkite/scripts/steps/functional/defend_workflows.sh
- label: 'Defend Workflows Cypress Tests'
- agents:
- queue: n2-4-virt
- depends_on: build
- timeout_in_minutes: 60
- parallelism: 16
- retry:
- automatic:
- - exit_status: '*'
- limit: 1
-
- - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
- label: 'Defend Workflows Cypress Tests on Serverless'
- agents:
- queue: n2-4-virt
- depends_on: build
- timeout_in_minutes: 60
- parallelism: 6
- retry:
- automatic:
- - exit_status: '*'
- limit: 1
-
  - command: .buildkite/scripts/steps/functional/threat_intelligence.sh
  label: 'Threat Intelligence Cypress Tests'
  agents:

.buildkite/pipelines/pull_request/defend_workflows.yml

@@ -0,0 +1,24 @@
+steps:
+ - command: .buildkite/scripts/steps/functional/defend_workflows.sh
+ label: 'Defend Workflows Cypress Tests'
+ agents:
+ queue: n2-4-virt
+ depends_on: build
+ timeout_in_minutes: 60
+ parallelism: 16
+ retry:
+ automatic:
+ - exit_status: '*'
+ limit: 1
+
+ - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
+ label: 'Defend Workflows Cypress Tests on Serverless'
+ agents:
+ queue: n2-4-virt
+ depends_on: build
+ timeout_in_minutes: 60
+ parallelism: 6
+ retry:
+ automatic:
+ - exit_status: '*'
+ limit: 1

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -179,6 +179,19 @@ const uploadPipeline = (pipelineContent: string | object) => {
  pipeline.push(getPipeline('.buildkite/pipelines/pull_request/cypress_burn.yml'));
  }
 
+ if (
+ (await doAnyChangesMatch([
+ /^packages\/kbn-securitysolution-.*/,
+ /^x-pack\/plugins\/security_solution/,
+ /^x-pack\/test\/defend_workflows_cypress/,
+ /^x-pack\/test\/security_solution_cypress/,
+ /^fleet_packages\.json/,
+ ])) ||
+ GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+ ) {
+ pipeline.push(getPipeline('.buildkite/pipelines/pull_request/defend_workflows.yml'));
+ }
+
  pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));
 
  // remove duplicated steps

.buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh

@@ -16,6 +16,9 @@ export JOB=kibana-security-solution-chrome
 
 buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true"
 
+mkdir .ftr
+retry 5 5 vault kv get -format=json -field=data secret/kibana-issues/dev/security-quality-gate/role-users > .ftr/role_users.json
+
 cd x-pack/test/security_solution_cypress
 set +e
 

.buildkite/scripts/serverless/create_deploy_tag/collect_commit_info.ts

@@ -13,6 +13,7 @@ import {
  getSelectedCommitHash,
  getCommitByHash,
  makeCommitInfoHtml,
+ getRecentCommits,
 } from './info_sections/commit_info';
 import {
  getArtifactBuild,
@@ -45,7 +46,8 @@ async function main() {
  const buildkiteBuild = await getOnMergePRBuild(selectedSha);
  const nextBuildContainingCommit = await getQAFBuildContainingCommit(
  selectedSha,
- selectedCommitInfo.date!
+ selectedCommitInfo.date!,
+ await getRecentCommits(50)
  );
  const artifactBuild = await getArtifactBuild(selectedSha);
  addBuildkiteInfoSection(

.buildkite/scripts/serverless/create_deploy_tag/create_deploy_tag.sh

@@ -11,9 +11,12 @@ fi
 
 echo "--- Creating deploy tag $DEPLOY_TAG at $KIBANA_COMMIT_SHA"
 
+echo "Fetching user identity from GitHub..."
+IDENTITY_JSON=$(ts-node .buildkite/scripts/serverless/create_deploy_tag/get_github_identity.ts)
+
 # Set git identity to whomever triggered the buildkite job
-git config user.email "$BUILDKITE_BUILD_CREATOR_EMAIL"
-git config user.name "$BUILDKITE_BUILD_CREATOR"
+git config user.email "${BUILDKITE_BUILD_CREATOR_EMAIL:-$(echo ${IDENTITY_JSON} | jq .email)}"
+git config user.name "${BUILDKITE_BUILD_CREATOR:-$(echo ${IDENTITY_JSON} | jq .name)}"
 
 # Create a tag for the deploy
 git tag -a "$DEPLOY_TAG" "$KIBANA_COMMIT_SHA" \

.buildkite/scripts/serverless/create_deploy_tag/get_github_identity.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+// Query the GitHub API for the user's GitHub identity
+import { octokit } from './shared';
+
+const getGitHubIdentity = async (): Promise<{
+ login: string;
+ name: string | null;
+ email: string | null;
+}> => {
+ const { data } = await octokit.users.getAuthenticated();
+
+ return {
+ login: data.login,
+ name: data.name,
+ email: data.email,
+ };
+};
+
+async function main() {
+ try {
+ const identity = await getGitHubIdentity();
+
+ if (!identity.name) {
+ identity.name = identity.login;
+ }
+
+ if (!identity.email) {
+ identity.email = `${identity.login}@elastic.co`;
+ }
+
+ return identity;
+ } catch (error) {
+ console.error(error);
+
+ return {
+ login: 'unknown-user',
+ name: 'Unknown User',
+ email: 'unknown-user@elastic.co',
+ };
+ }
+}
+
+if (require.main === module) {
+ main().then((identity) => {
+ console.log(JSON.stringify(identity));
+ });
+} else {
+ module.exports = main;
+}

.buildkite/scripts/serverless/create_deploy_tag/info_sections/build_info.ts

@@ -6,8 +6,8 @@
  * Side Public License, v 1.
  */
 
-import { components } from '@octokit/openapi-types';
-import { buildkite, buildkiteBuildStateToEmoji, CommitWithStatuses, octokit } from '../shared';
+import { buildkite, buildkiteBuildStateToEmoji, CommitWithStatuses } from '../shared';
+import { GitCommitExtract } from './commit_info';
 import { Build } from '#pipeline-utils/buildkite';
 
 const QA_FTR_TEST_SLUG = 'appex-qa-serverless-kibana-ftr-tests';
@@ -70,27 +70,30 @@ export async function getArtifactBuild(commitSha: string): Promise<BuildkiteBuil
 
 export async function getQAFBuildContainingCommit(
  commitSha: string,
- date: string
+ date: string,
+ recentGitCommits: GitCommitExtract[]
 ): Promise<BuildkiteBuildExtract | null> {
- // List of commits
- const commitShaList = await getCommitListCached();
-
  // List of QAF builds
  const qafBuilds = await buildkite.getBuildsAfterDate(QA_FTR_TEST_SLUG, date, 30);
+ qafBuilds.reverse();
 
  // Find the first build that contains this commit
- const build = qafBuilds.find((kbBuild) => {
- // Check if build.commit is after commitSha?
- const kibanaCommitSha = tryGetKibanaBuildHashFromQAFBuild(kbBuild);
- const buildkiteBuildShaIndex = commitShaList.findIndex((c) => c.sha === kibanaCommitSha);
- const commitShaIndex = commitShaList.findIndex((c) => c.sha === commitSha);
-
- return (
- commitShaIndex !== -1 &&
- buildkiteBuildShaIndex !== -1 &&
- buildkiteBuildShaIndex < commitShaIndex
- );
- });
+ const build = qafBuilds
+ // Only search across scheduled builds, triggered builds might run with different commits
+ .filter((e) => e.source === 'schedule')
+ .find((kbBuild) => {
+ const commitShaIndex = recentGitCommits.findIndex((c) => c.sha === commitSha);
+
+ const kibanaCommitOfFTR = tryGetKibanaBuildHashFromQAFBuild(kbBuild);
+ const buildkiteBuildShaIndex = recentGitCommits.findIndex((c) => c.sha === kibanaCommitOfFTR);
+
+ // Check if build.commit is after commitSha?
+ return (
+ commitShaIndex !== -1 &&
+ buildkiteBuildShaIndex !== -1 &&
+ buildkiteBuildShaIndex <= commitShaIndex
+ );
+ });
 
  if (!build) {
  return null;
@@ -121,25 +124,6 @@ function tryGetKibanaBuildHashFromQAFBuild(build: Build) {
  }
 }
 
-let _commitListCache: Array<components['schemas']['commit']> | null = null;
-async function getCommitListCached() {
- if (!_commitListCache) {
- const resp = await octokit.request<'GET /repos/{owner}/{repo}/commits'>(
- 'GET /repos/{owner}/{repo}/commits',
- {
- owner: 'elastic',
- repo: 'kibana',
- headers: {
- accept: 'application/vnd.github.v3+json',
- 'X-GitHub-Api-Version': '2022-11-28',
- },
- }
- );
- _commitListCache = resp.data;
- }
- return _commitListCache;
-}
-
 function makeBuildInfoSnippetHtml(name: string, build: BuildkiteBuildExtract | null) {
  if (!build) {
  return `[❓] ${name} - no build found`;

.buildkite/scripts/serverless/create_deploy_tag/list_commit_candidates.ts

@@ -83,7 +83,7 @@ async function enrichWithStatuses(commits: GitCommitExtract[]): Promise<CommitWi
  };
  }
 
- const nextFTRBuild = await getQAFBuildContainingCommit(commit.sha, commit.date);
+ const nextFTRBuild = await getQAFBuildContainingCommit(commit.sha, commit.date, commits);
  const artifactBuild = await getArtifactBuild(commit.sha);
 
  return {

.buildkite/scripts/serverless/create_deploy_tag/mock_exec.ts

@@ -75,7 +75,10 @@ const loadFakeResponses = (() => {
 })();
 
 const makeMockExec = (id: string) => {
- console.warn("--- Using mock exec, don't use this on CI. ---");
+ if (process.env?.CI?.match(/(1|true)/i)) {
+ throw new Error(`Mock exec is not supported in CI - your commands won't be executed.`);
+ }
+
  const callStorage = getCallStorage();
  const calls = callStorage[id];
 

.buildkite/scripts/steps/artifacts/trigger.sh

@@ -12,9 +12,5 @@ echo "--- Trigger artifact builds"
 if [ "$BEATS_MANIFEST_LATEST_URL" = "$KIBANA_BEATS_MANIFEST_URL" ]; then
  echo "Kibana has the latest version of beats, skipping trigger"
 else
- # Staging builds are not necessary on main
- if [[ "$BUILDKITE_BRANCH" != "main" ]]; then
- ts-node .buildkite/scripts/steps/trigger_pipeline.ts kibana-artifacts-staging "$BUILDKITE_BRANCH"
- fi
- ts-node .buildkite/scripts/steps/trigger_pipeline.ts kibana-artifacts-snapshot "$BUILDKITE_BRANCH"
+ ts-node .buildkite/scripts/steps/trigger_pipeline.ts kibana-artifacts-staging "$BUILDKITE_BRANCH"
 fi