Skip to content

Commit bb0f434

Browse files
michel-latermanmichel-laterman
committed
Merge remote-tracking branch 'origin/main' into integration-fips-status-es
2 parents 7db3303 + c1be040 commit bb0f434

File tree

132 files changed

+39008
-22249
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

132 files changed

+39008
-22249
lines changed

‎.buildkite/bk.integration-fips.pipeline.yml‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
11
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
22

33
env:
4-
DOCKER_REGISTRY: "docker.elastic.co"
54
ASDF_MAGE_VERSION: 1.14.0
65

7-
IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1749258065"
8-
IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1748955449"
9-
IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1748955449"
6+
IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1749862860"
7+
IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1749862860"
8+
IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1749862860"
109

1110
steps:
1211
- label: Build and push custom elastic-agent image
@@ -16,16 +15,19 @@ steps:
1615
env:
1716
FIPS: "true"
1817
CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
19-
CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud"
20-
TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud:git-${BUILDKITE_COMMIT:0:12}"
18+
CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips"
19+
TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
2120
command: |
22-
buildkite-agent artifact download build/distributions/elastic-agent-fips-cloud-*-linux-amd64.docker.tar.gz . --step 'packaging-containers-x86-64-fips'
21+
buildkite-agent artifact download build/distributions/elastic-agent-cloud-fips-*-linux-amd64.docker.tar.gz . --step 'packaging-containers-x86-64-fips'
2322
mage cloud:load
2423
mage cloud:push
2524
agents:
2625
provider: "gcp"
2726
machineType: "n1-standard-8"
2827
image: "${IMAGE_UBUNTU_2404_X86_64}"
28+
plugins:
29+
- elastic/vault-docker-login#v0.5.2:
30+
secret_path: 'kv/ci-shared/platform-ingest/elastic_docker_registry'
2931

3032
- label: Start ESS stack for FIPS integration tests
3133
key: integration-fips-ess
@@ -34,8 +36,8 @@ steps:
3436
env:
3537
ASDF_TERRAFORM_VERSION: 1.9.2
3638
CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
37-
CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud"
38-
TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud:git-${BUILDKITE_COMMIT:0:12}"
39+
CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips"
40+
TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
3941
command: |
4042
source .buildkite/scripts/steps/ess_start.sh
4143
artifact_paths:
@@ -56,8 +58,8 @@ steps:
5658
env:
5759
FIPS: "true"
5860
CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
59-
CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud"
60-
TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud:git-${BUILDKITE_COMMIT:0:12}"
61+
CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips"
62+
TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
6163
command: |
6264
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64-fips'
6365
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix.groups}} {{matrix.sudo}}
@@ -85,8 +87,8 @@ steps:
8587
env:
8688
FIPS: "true"
8789
CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
88-
CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud"
89-
TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud:git-${BUILDKITE_COMMIT:0:12}"
90+
CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips"
91+
TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
9092
command: |
9193
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64-fips'
9294
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix.groups}} {{matrix.sudo}}

‎.buildkite/bk.integration.pipeline.yml‎

Lines changed: 133 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,49 @@
11
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
22

33
env:
4-
DOCKER_REGISTRY: "docker.elastic.co"
54
VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
65
ASDF_MAGE_VERSION: 1.14.0
76

87
# The following images are defined here and their values will be updated by updatecli
98
# Please do not change them manually.
10-
IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1749258065"
11-
IMAGE_UBUNTU_2404_ARM_64: "platform-ingest-elastic-agent-ubuntu-2404-aarch64-1749258065"
12-
IMAGE_RHEL_8: "platform-ingest-elastic-agent-rhel-8-1749258065"
13-
IMAGE_DEBIAN_12: "platform-ingest-elastic-agent-debian-12-1749258065"
14-
IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1749258065"
15-
IMAGE_WIN_2025: "platform-ingest-elastic-agent-windows-2025-1749258065"
9+
IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1749862860"
10+
IMAGE_UBUNTU_2404_ARM_64: "platform-ingest-elastic-agent-ubuntu-2404-aarch64-1749862860"
11+
IMAGE_RHEL_8: "platform-ingest-elastic-agent-rhel-8-1749862860"
12+
IMAGE_DEBIAN_12: "platform-ingest-elastic-agent-debian-12-1749862860"
13+
IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1749862860"
14+
IMAGE_WIN_2025: "platform-ingest-elastic-agent-windows-2025-1749862860"
15+
16+
# This section is used to define the plugins that will be used in the pipeline.
17+
# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
18+
common:
19+
- google_oidc_plugin: &google_oidc_plugin
20+
# See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/elastic-agent/01-gcp-oidc.tf
21+
# This plugin authenticates to Google Cloud using the OIDC token.
22+
elastic/oblt-google-auth#v1.3.0:
23+
lifetime: 10800 # seconds
24+
project-id: "elastic-observability-ci"
25+
project-number: "911195782929"
26+
# see https://github.com/avaly/gcp-secret-manager-buildkite-plugin/pull/10
27+
# see https://github.com/avaly/gcp-secret-manager-buildkite-plugin/pull/11
28+
# - gcp_serverless_secrets_plugin: &gcp_serverless_secrets_plugin
29+
#avaly/gcp-secret-manager#v1.2.0:
30+
- gcp_serverless_secrets_plugin: &gcp_serverless_secrets_plugin
31+
elastic/gcp-secret-manager#v1.3.0-elastic:
32+
env:
33+
# These secrets are created in .github/workflows/serverless-project.yml
34+
ELASTICSEARCH_HOST: ea-serverless-it-elasticsearch-hostname
35+
ELASTICSEARCH_PASSWORD: ea-serverless-it-elasticsearch-password
36+
ELASTICSEARCH_USERNAME: ea-serverless-it-elasticsearch-username
37+
KIBANA_HOST: ea-serverless-it-kibana-hostname
38+
KIBANA_USERNAME: ea-serverless-it-kibana-username
39+
KIBANA_PASSWORD: ea-serverless-it-kibana-password
1640

1741
steps:
1842
- label: Start ESS stack for integration tests
1943
key: integration-ess
44+
notify:
45+
- github_commit_status:
46+
context: "buildkite/elastic-agent-extended-testing - ESS stack provision"
2047
env:
2148
ASDF_TERRAFORM_VERSION: 1.9.2
2249
command: |
@@ -32,17 +59,20 @@ steps:
3259

3360
- group: "Extended runtime leak tests"
3461
key: extended-integration-tests
62+
notify:
63+
- github_commit_status:
64+
context: "buildkite/elastic-agent-extended-testing - Runtime leak tests"
3565
depends_on:
3666
- integration-ess
3767
steps:
3868
- label: "Windows:2022:amd64:sudo"
3969
depends_on:
4070
- packaging-windows
4171
env:
42-
TEST_LONG_RUNNING: "true"
72+
TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/leak"
4373
command: |
4474
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
45-
.buildkite/scripts/integration-tests.ps1 fleet true TestLongRunningAgentForLeaks
75+
.buildkite/scripts/steps/integration_tests_tf.ps1 fleet true
4676
artifact_paths:
4777
- build/**
4878
- build/diagnostics/**
@@ -57,10 +87,10 @@ steps:
5787
depends_on:
5888
- packaging-windows
5989
env:
60-
TEST_LONG_RUNNING: "true"
90+
TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/leak"
6191
command: |
6292
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
63-
.buildkite/scripts/integration-tests.ps1 fleet true TestLongRunningAgentForLeaks
93+
.buildkite/scripts/steps/integration_tests_tf.ps1 fleet true
6494
artifact_paths:
6595
- build/**
6696
- build/diagnostics/**
@@ -74,10 +104,10 @@ steps:
74104
- label: "Ubuntu:2404:amd64:sudo"
75105
depends_on: packaging-ubuntu-x86-64
76106
env:
77-
TEST_LONG_RUNNING: "true"
107+
TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/leak"
78108
command: |
79109
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64'
80-
.buildkite/scripts/steps/integration_tests_tf.sh fleet true TestLongRunningAgentForLeaks
110+
.buildkite/scripts/steps/integration_tests_tf.sh fleet true
81111
artifact_paths:
82112
- build/**
83113
- build/diagnostics/**
@@ -91,6 +121,9 @@ steps:
91121

92122
- group: "Stateful: Windows"
93123
key: integration-tests-win
124+
notify:
125+
- github_commit_status:
126+
context: "buildkite/elastic-agent-extended-testing - Windows"
94127
depends_on:
95128
- integration-ess
96129
steps:
@@ -99,7 +132,7 @@ steps:
99132
- packaging-windows
100133
command: |
101134
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
102-
.buildkite/scripts/integration-tests.ps1 {{matrix}} true
135+
.buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} true
103136
artifact_paths:
104137
- build/**
105138
- build/diagnostics/**
@@ -125,7 +158,7 @@ steps:
125158
- packaging-windows
126159
command: |
127160
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
128-
.buildkite/scripts/integration-tests.ps1 {{matrix}} false
161+
.buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} false
129162
artifact_paths:
130163
- build/**
131164
- build/diagnostics/**
@@ -144,7 +177,7 @@ steps:
144177
- packaging-windows
145178
command: |
146179
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
147-
.buildkite/scripts/integration-tests.ps1 {{matrix}} true
180+
.buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} true
148181
artifact_paths:
149182
- build/**
150183
- build/diagnostics/**
@@ -170,7 +203,7 @@ steps:
170203
- packaging-windows
171204
command: |
172205
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
173-
.buildkite/scripts/integration-tests.ps1 {{matrix}} false
206+
.buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} false
174207
artifact_paths:
175208
- build/**
176209
- build/diagnostics/**
@@ -186,6 +219,9 @@ steps:
186219

187220
- group: "Stateful:Ubuntu"
188221
key: integration-tests-ubuntu
222+
notify:
223+
- github_commit_status:
224+
context: "buildkite/elastic-agent-extended-testing - Ubuntu"
189225
depends_on:
190226
- integration-ess
191227
steps:
@@ -295,6 +331,9 @@ steps:
295331

296332
- group: "Stateful:Debian"
297333
key: integration-tests-debian
334+
notify:
335+
- github_commit_status:
336+
context: "buildkite/elastic-agent-extended-testing - Debian"
298337
depends_on:
299338
- integration-ess
300339
steps:
@@ -352,6 +391,9 @@ steps:
352391

353392
- group: "Stateful(Sudo):RHEL8"
354393
key: integration-tests-rhel8
394+
notify:
395+
- github_commit_status:
396+
context: "buildkite/elastic-agent-extended-testing - RHEL8"
355397
depends_on:
356398
- integration-ess
357399
steps:
@@ -374,6 +416,9 @@ steps:
374416

375417
- group: "Kubernetes"
376418
key: integration-tests-kubernetes
419+
notify:
420+
- github_commit_status:
421+
context: "buildkite/elastic-agent-extended-testing - Kubernetes"
377422
depends_on:
378423
- integration-ess
379424
- packaging-containers-x86-64
@@ -417,6 +462,76 @@ steps:
417462
- v1.32.0
418463
- v1.33.0
419464

465+
- group: "Serverless integration test"
466+
key: integration-tests-serverless
467+
notify:
468+
- github_commit_status:
469+
context: "buildkite/elastic-agent-extended-testing - Serverless integration test"
470+
steps:
471+
- label: "Windows:2022:amd64:sudo"
472+
depends_on:
473+
- packaging-windows
474+
env:
475+
TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/serverless"
476+
command: |
477+
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
478+
.buildkite/scripts/buildkite-integration-tests.ps1 fleet true
479+
artifact_paths:
480+
- build/**
481+
- build/diagnostics/**
482+
retry:
483+
automatic:
484+
limit: 1
485+
agents:
486+
provider: "gcp"
487+
machineType: "n2-standard-8"
488+
image: "${IMAGE_WIN_2022}"
489+
plugins:
490+
- *google_oidc_plugin
491+
- *gcp_serverless_secrets_plugin
492+
493+
- label: "Windows:2025:amd64:sudo"
494+
depends_on:
495+
- packaging-windows
496+
env:
497+
TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/serverless"
498+
command: |
499+
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
500+
.buildkite/scripts/buildkite-integration-tests.ps1 fleet true
501+
artifact_paths:
502+
- build/**
503+
- build/diagnostics/**
504+
retry:
505+
automatic:
506+
limit: 1
507+
agents:
508+
provider: "gcp"
509+
machineType: "n2-standard-8"
510+
image: "${IMAGE_WIN_2025}"
511+
plugins:
512+
- *google_oidc_plugin
513+
- *gcp_serverless_secrets_plugin
514+
- label: "Ubuntu:2404:amd64:sudo"
515+
depends_on: packaging-ubuntu-x86-64
516+
env:
517+
TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/serverless"
518+
command: |
519+
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64'
520+
sudo -E .buildkite/scripts/buildkite-integration-tests.sh fleet true
521+
artifact_paths:
522+
- build/**
523+
- build/diagnostics/**
524+
retry:
525+
automatic:
526+
limit: 1
527+
agents:
528+
provider: "gcp"
529+
machineType: "n2-standard-8"
530+
image: "${IMAGE_UBUNTU_2404_X86_64}"
531+
plugins:
532+
- *google_oidc_plugin
533+
- *gcp_serverless_secrets_plugin
534+
420535
- label: ESS stack cleanup
421536
depends_on:
422537
- integration-tests-ubuntu
@@ -441,6 +556,7 @@ steps:
441556
- integration-tests-win
442557
- integration-tests-rhel8
443558
- integration-tests-kubernetes
559+
- integration-tests-serverless
444560
allow_dependency_failure: true
445561
command: |
446562
buildkite-agent artifact download "build/*.xml" .

0 commit comments

Comments
 (0)