From 68336b4ccf283e6f22b1588db4b88dfa724dc28c Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 25 Apr 2024 12:36:22 +0300 Subject: [PATCH 1/9] Fixes for packaging pipeline --- .buildkite/packaging.pipeline.yml | 14 +++++++------- .buildkite/scripts/dra.sh | 18 +++++++++++++++++- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index fb396328d8e..6570db22603 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -16,7 +16,7 @@ steps: key: dashboards steps: - label: Snapshot dashboards - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true key: dashboards-snapshot # TODO: container with go and make agents: @@ -46,7 +46,7 @@ steps: - build/distributions/**/* - group: Packaging snapshot - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true key: packaging-snapshot steps: # x86 @@ -123,7 +123,7 @@ steps: key: packaging-staging ## Only for release - if: "build.branch =~ /\\d+\\.\\d+/" + if: build.branch =~ /\d+\.\d+/ steps: # x86 - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" @@ -181,10 +181,10 @@ steps: - x-pack/packetbeat ## Agentbeat needs more CPUs because it builds many other beats - - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" env: PLATFORMS: "${PLATFORMS_AMD}" - SNAPSHOT: true + SNAPSHOT: false command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: gcp @@ -200,7 +200,7 @@ steps: steps: - label: DRA Snapshot ## Only for release branches and main - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true key: dra-snapshot env: DRA_WORKFLOW: snapshot @@ -216,7 +216,7 @@ steps: - label: DRA Staging ## Only for release branches - if: "build.branch =~ /\\d+\\.\\d+/" + if: build.branch =~ /\d+\.\d+/ key: dra-staging env: DRA_WORKFLOW: staging diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index ee0f4b286f0..4957ef36199 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -28,6 +28,22 @@ release_manager_login echo "+++ Changing permissions for the BK API commands" sudo chown -R :1000 build/distributions/ +echo "+++ :hammer_and_pick: Listing $BRANCH $DRA_WORKFLOW DRA artifacts..." +docker run --rm \ + --name release-manager \ + -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ + -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \ + -e VAULT_SECRET_ID="${VAULT_SECRET}" \ + --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli list \ + --project "beats" \ + --branch "${BRANCH}" \ + --commit "${BUILDKITE_COMMIT}" \ + --workflow "${DRA_WORKFLOW}" \ + --version "${BEAT_VERSION}" \ + --artifact-set "main" + echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..." docker run --rm \ --name release-manager \ @@ -43,4 +59,4 @@ docker run --rm \ --workflow "${DRA_WORKFLOW}" \ --version "${BEAT_VERSION}" \ --artifact-set "main" \ - ${DRY_RUN} \ No newline at end of file + ${DRY_RUN} From 12bd2342a3970fc9b9def3b7bf2966e2b7580bbf Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 25 Apr 2024 12:41:31 +0300 Subject: [PATCH 2/9] bug fix --- .buildkite/packaging.pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 6570db22603..f5142b68477 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -32,7 +32,7 @@ steps: - build/distributions/**/* - label: Staging dashboards - if: "build.branch =~ /\\d+\\.\\d+/" + if: build.branch =~ /\d+\.\d+/ key: dashboards-staging # TODO: container with go and make agents: From 4489aecff02479b69d275e6338d9843b17f12461 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 25 Apr 2024 12:44:07 +0300 Subject: [PATCH 3/9] another bug --- .buildkite/packaging.pipeline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index f5142b68477..a2c69ead3a0 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -16,7 +16,7 @@ steps: key: dashboards steps: - label: Snapshot dashboards - if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true + if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: dashboards-snapshot # TODO: container with go and make agents: @@ -46,7 +46,7 @@ steps: - build/distributions/**/* - group: Packaging snapshot - if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true + if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: packaging-snapshot steps: # x86 @@ -200,7 +200,7 @@ steps: steps: - label: DRA Snapshot ## Only for release branches and main - if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true + if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: dra-snapshot env: DRA_WORKFLOW: snapshot From 5f1079292440653c426acc586e69e96614dd7f5f Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 25 Apr 2024 12:57:11 +0300 Subject: [PATCH 4/9] more fixes --- .buildkite/packaging.pipeline.yml | 34 ++++++++------------- .buildkite/scripts/packaging/package-dra.sh | 2 +- 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index a2c69ead3a0..471b9abaaf2 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -8,7 +8,7 @@ env: GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - PLATFORMS_AMD: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" PLATFORMS_ARM: "linux/arm64" steps: @@ -49,10 +49,9 @@ steps: if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: packaging-snapshot steps: - # x86 - - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + - label: "SNAPSHOT: {{matrix}}/Packaging cross architecture artifacts" env: - PLATFORMS: "${PLATFORMS_AMD}" + PLATFORMS: "${PLATFORMS}" SNAPSHOT: true command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: @@ -78,8 +77,7 @@ steps: - x-pack/packetbeat - x-pack/winlogbeat - ## ARM - - label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot" + - label: "SNAPSHOT: {{matrix}}/Packaging docker artifacts for Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" PACKAGES: "docker" @@ -105,19 +103,17 @@ steps: - x-pack/packetbeat ## Agentbeat needs more CPUs because it builds many other beats - - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + - label: "SNAPSHOT: x-pack/agentbeat/Packaging cross architecture artifacts" env: - PLATFORMS: "${PLATFORMS_AMD}" + PLATFORMS: "${PLATFORMS}" SNAPSHOT: true - command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat" agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" artifact_paths: - build/distributions/**/* - matrix: - - x-pack/agentbeat - group: Packaging staging @@ -125,10 +121,9 @@ steps: ## Only for release if: build.branch =~ /\d+\.\d+/ steps: - # x86 - - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" + - label: "STAGING: {{matrix}}/Packaging cross architecture artifacts" env: - PLATFORMS: "${PLATFORMS_AMD}" + PLATFORMS: "${PLATFORMS}" SNAPSHOT: false command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: @@ -154,8 +149,7 @@ steps: - x-pack/packetbeat - x-pack/winlogbeat - ## ARM - - label: ":linux: {{matrix}}/Packaging Linux arm64 Staging" + - label: "STAGING: {{matrix}}/Packaging docker artifacts for Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" PACKAGES: "docker" @@ -181,19 +175,17 @@ steps: - x-pack/packetbeat ## Agentbeat needs more CPUs because it builds many other beats - - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" + - label: "STAGING: x-pack/agentbeat/Packaging cross architecture artifacts" env: - PLATFORMS: "${PLATFORMS_AMD}" + PLATFORMS: "${PLATFORMS}" SNAPSHOT: false - command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat" agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" artifact_paths: - build/distributions/**/* - matrix: - - x-pack/agentbeat - group: DRA publish key: dra diff --git a/.buildkite/scripts/packaging/package-dra.sh b/.buildkite/scripts/packaging/package-dra.sh index a70bd455278..10d5425da36 100755 --- a/.buildkite/scripts/packaging/package-dra.sh +++ b/.buildkite/scripts/packaging/package-dra.sh @@ -7,7 +7,7 @@ if [ -z "$BEAT_DIR" ]; then exit 1 fi -echo "~~~Packaging : $BEAT_DIR" +echo "~~~ Packaging : $BEAT_DIR" WORKSPACE=$(pwd) BEAT_NAME_SLUG=$(echo "$BEAT_DIR" | sed 's/x-pack\///g') From acbc1a7df4b0fb9baed716ceab014defc674c076 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 25 Apr 2024 13:04:33 +0300 Subject: [PATCH 5/9] fix GCP instance type --- .buildkite/packaging.pipeline.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 471b9abaaf2..44361159bac 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -22,7 +22,7 @@ steps: agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" commands: - make build/distributions/dependencies.csv - make beats-dashboards @@ -38,7 +38,7 @@ steps: agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" commands: - make build/distributions/dependencies.csv - make beats-dashboards @@ -57,7 +57,7 @@ steps: agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: - build/distributions/**/* matrix: @@ -129,7 +129,7 @@ steps: agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: - build/distributions/**/* matrix: From 89f6d0c6753d67431929b49245d4178c2b925b1b Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 25 Apr 2024 13:14:08 +0300 Subject: [PATCH 6/9] aesthetic changes --- .buildkite/packaging.pipeline.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 44361159bac..19e435acd1e 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -49,7 +49,7 @@ steps: if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: packaging-snapshot steps: - - label: "SNAPSHOT: {{matrix}}/Packaging cross architecture artifacts" + - label: "{{matrix}} Packaging" env: PLATFORMS: "${PLATFORMS}" SNAPSHOT: true @@ -77,7 +77,7 @@ steps: - x-pack/packetbeat - x-pack/winlogbeat - - label: "SNAPSHOT: {{matrix}}/Packaging docker artifacts for Linux/arm64" + - label: "{{matrix}} Packaging docker Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" PACKAGES: "docker" @@ -103,7 +103,7 @@ steps: - x-pack/packetbeat ## Agentbeat needs more CPUs because it builds many other beats - - label: "SNAPSHOT: x-pack/agentbeat/Packaging cross architecture artifacts" + - label: "x-pack/agentbeat Packaging" env: PLATFORMS: "${PLATFORMS}" SNAPSHOT: true @@ -115,13 +115,13 @@ steps: artifact_paths: - build/distributions/**/* - - group: Packaging staging + - group: Packaging Staging key: packaging-staging ## Only for release if: build.branch =~ /\d+\.\d+/ steps: - - label: "STAGING: {{matrix}}/Packaging cross architecture artifacts" + - label: "{{matrix}} Packaging" env: PLATFORMS: "${PLATFORMS}" SNAPSHOT: false @@ -149,7 +149,7 @@ steps: - x-pack/packetbeat - x-pack/winlogbeat - - label: "STAGING: {{matrix}}/Packaging docker artifacts for Linux/arm64" + - label: "{{matrix}} Packaging docker Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" PACKAGES: "docker" @@ -175,7 +175,7 @@ steps: - x-pack/packetbeat ## Agentbeat needs more CPUs because it builds many other beats - - label: "STAGING: x-pack/agentbeat/Packaging cross architecture artifacts" + - label: "x-pack/agentbeat Packaging" env: PLATFORMS: "${PLATFORMS}" SNAPSHOT: false From 9bc5bcbd784ce92e3beb4df88a7181524fabe52b Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 25 Apr 2024 13:18:00 +0300 Subject: [PATCH 7/9] more aesthetic fixes --- .buildkite/packaging.pipeline.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 19e435acd1e..3cc2698fc9c 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -49,7 +49,7 @@ steps: if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: packaging-snapshot steps: - - label: "{{matrix}} Packaging" + - label: "SNAPSHOT: {{matrix}}" env: PLATFORMS: "${PLATFORMS}" SNAPSHOT: true @@ -77,7 +77,7 @@ steps: - x-pack/packetbeat - x-pack/winlogbeat - - label: "{{matrix}} Packaging docker Linux/arm64" + - label: "SNAPSHOT: {{matrix}} docker Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" PACKAGES: "docker" @@ -103,7 +103,7 @@ steps: - x-pack/packetbeat ## Agentbeat needs more CPUs because it builds many other beats - - label: "x-pack/agentbeat Packaging" + - label: "SNAPSHOT: x-pack/agentbeat" env: PLATFORMS: "${PLATFORMS}" SNAPSHOT: true @@ -121,7 +121,7 @@ steps: ## Only for release if: build.branch =~ /\d+\.\d+/ steps: - - label: "{{matrix}} Packaging" + - label: "STAGING: {{matrix}}" env: PLATFORMS: "${PLATFORMS}" SNAPSHOT: false @@ -149,7 +149,7 @@ steps: - x-pack/packetbeat - x-pack/winlogbeat - - label: "{{matrix}} Packaging docker Linux/arm64" + - label: "STAGING: {{matrix}} docker Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" PACKAGES: "docker" @@ -175,7 +175,7 @@ steps: - x-pack/packetbeat ## Agentbeat needs more CPUs because it builds many other beats - - label: "x-pack/agentbeat Packaging" + - label: "STAGING: x-pack/agentbeat" env: PLATFORMS: "${PLATFORMS}" SNAPSHOT: false From 2e212b001e609757fad634c35c008ec4c6fcdd6f Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 25 Apr 2024 14:07:44 +0300 Subject: [PATCH 8/9] use custom vm image for publish --- .buildkite/packaging.pipeline.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 3cc2698fc9c..9ea18eb62b5 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -204,7 +204,9 @@ steps: .buildkite/scripts/packaging/prepare-release-manager.sh .buildkite/scripts/dra.sh agents: - provider: "gcp" + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - label: DRA Staging ## Only for release branches @@ -220,4 +222,6 @@ steps: .buildkite/scripts/packaging/prepare-release-manager.sh .buildkite/scripts/dra.sh agents: - provider: "gcp" + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" From 8faf07f233edc55cc089b965883592ccce8848c2 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 25 Apr 2024 15:53:46 +0300 Subject: [PATCH 9/9] Fix regex --- .buildkite/packaging.pipeline.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 9ea18eb62b5..63ea44795ef 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -16,7 +16,7 @@ steps: key: dashboards steps: - label: Snapshot dashboards - if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: dashboards-snapshot # TODO: container with go and make agents: @@ -32,7 +32,7 @@ steps: - build/distributions/**/* - label: Staging dashboards - if: build.branch =~ /\d+\.\d+/ + if: build.branch =~ /^\d+\.\d+$/ key: dashboards-staging # TODO: container with go and make agents: @@ -46,7 +46,7 @@ steps: - build/distributions/**/* - group: Packaging snapshot - if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: packaging-snapshot steps: - label: "SNAPSHOT: {{matrix}}" @@ -119,7 +119,7 @@ steps: key: packaging-staging ## Only for release - if: build.branch =~ /\d+\.\d+/ + if: build.branch =~ /^\d+\.\d+$/ steps: - label: "STAGING: {{matrix}}" env: @@ -192,7 +192,7 @@ steps: steps: - label: DRA Snapshot ## Only for release branches and main - if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: dra-snapshot env: DRA_WORKFLOW: snapshot @@ -210,7 +210,7 @@ steps: - label: DRA Staging ## Only for release branches - if: build.branch =~ /\d+\.\d+/ + if: build.branch =~ /^\d+\.\d+$/ key: dra-staging env: DRA_WORKFLOW: staging