Merge branch 'main' into cap-processor

.buildkite/auditbeat/auditbeat-pipeline.yml

@@ -1,13 +1,18 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 env:
-  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
-  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
+  BEATS_PROJECT_NAME: "auditbeat"
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
   IMAGE_WIN_2016: "family/core-windows-2016"
   IMAGE_WIN_2019: "family/core-windows-2019"
   IMAGE_WIN_2022: "family/core-windows-2022"
-  IMAGE_RHEL9: "family/core-rhel-9"
+  IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+  GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
 
 steps:
   - group: "Auditbeat Mandatory Testing"
@@ -24,6 +29,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
         artifact_paths:
           - "auditbeat/build/*.xml"
           - "auditbeat/build/*.json"
@@ -37,19 +43,20 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
         artifact_paths:
           - "auditbeat/build/*.xml"
           - "auditbeat/build/*.json"
 
       - label: ":windows:-{{matrix.image}} Unit Tests"
-        command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
         notify:
           - github_commit_status:
               context: "Auditbeat: windows/Unit Tests"
         agents:
           provider: "gcp"
           image: "{{matrix.image}}"
-          machine_type: "n2-standard-8"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 200
           disk_type: "pd-ssd"
         matrix:
@@ -72,6 +79,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
 
   - group: "Extended Testing"
     key: "extended-tests"
@@ -88,8 +96,8 @@ steps:
               context: "Auditbeat/Extended: Unit Tests ARM"
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
-          instanceType: "t4g.large"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         artifact_paths: "auditbeat/build/*.xml"
 
       - label: ":mac: MacOS Unit Tests"
@@ -112,14 +120,14 @@ steps:
     steps:
       - label: ":windows: Win 2019 Unit Tests"
         key: "win-extended-2019"
-        command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
         notify:
           - github_commit_status:
               context: "Auditbeat/Extended: Win-2019 Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
-          machine_type: "n2-standard-8"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 200
           disk_type: "pd-ssd"
         artifact_paths:
@@ -134,4 +142,4 @@ steps:
 
     steps:
       - label: Package pipeline
-        commands: ".buildkite/auditbeat/scripts/package-step.sh"
+        commands: ".buildkite/scripts/packaging/package-step.sh"

.buildkite/auditbeat/scripts/crosscompile.sh

@@ -2,7 +2,5 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/linux-env.sh
-
 echo "--- Executing Crosscompile"
 make -C auditbeat crosscompile

.buildkite/auditbeat/scripts/unit-tests.sh

@@ -2,11 +2,8 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/linux-env.sh
-
 echo "--- Running Unit Tests"
 sudo chmod -R go-w auditbeat/
 
-cd auditbeat
 umask 0022
-mage build unitTest
+mage -d auditbeat build unitTest

.buildkite/buildkite.yml

@@ -0,0 +1,44 @@
+projects:
+    - "auditbeat"
+    - "deploy/kubernetes"
+    - "filebeat"
+    - "heartbeat"
+    - "libbeat"
+    - "metricbeat"
+    - "packetbeat"
+    - "winlogbeat"
+    - "x-pack/auditbeat"
+    - "x-pack/dockerlogbeat"
+    - "x-pack/filebeat"
+    - "x-pack/functionbeat"
+    - "x-pack/heartbeat"
+    - "x-pack/libbeat"
+    - "x-pack/metricbeat"
+    - "x-pack/osquerybeat"
+    - "x-pack/packetbeat"
+    - "x-pack/winlogbeat"
+
+## Changeset macros that are defined here and used in each specific 3.0 pipeline.
+changeset:
+    ci:
+        - "^Jenkinsfile"
+        - "^\\.ci/scripts/.*"
+    oss:
+        - "^go.mod"
+        - "^pytest.ini"
+        - "^dev-tools/.*"
+        - "^libbeat/.*"
+        - "^testing/.*"
+    xpack:
+        - "^go.mod"
+        - "^pytest.ini"
+        - "^dev-tools/.*"
+        - "^libbeat/.*"
+        - "^testing/.*"
+        - "^x-pack/libbeat/.*"
+
+disabled:
+    when:
+        labels: ## Skip the GitHub Pull Request builds if any of the given GitHub labels match with the assigned labels in the PR.
+            - skip-ci
+        draft: true  ## Skip the GitHub Pull Request builds with Draft PRs.

.buildkite/env-scripts/env.sh

@@ -1,19 +1,53 @@
 #!/usr/bin/env bash
 
-SETUP_GVM_VERSION="v0.5.1"
+source .buildkite/env-scripts/util.sh
+
+DOCS_CHANGESET="^.*\.(asciidoc|md)$
+deploy/kubernetes/.*-kubernetes.yaml"
+PACKAGING_CHANGESET="^dev-tools/packaging/
+^.go-version"
+
+REPO="beats"
 WORKSPACE="$(pwd)"
 BIN="${WORKSPACE}/bin"
 HW_TYPE="$(uname -m)"
 PLATFORM_TYPE="$(uname)"
-REPO="beats"
 TMP_FOLDER="tmp.${REPO}"
+SNAPSHOT="true"
+PYTEST_ADDOPTS=""
+OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
+XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
+
+SETUP_GVM_VERSION="v0.5.1"
+ASDF_MAGE_VERSION="1.14.0"
+SETUP_WIN_PYTHON_VERSION="3.11.0"
+
+# Docker & DockerHub
+DOCKER_COMPOSE_VERSION="1.21.0"
 DOCKER_REGISTRY="docker.elastic.co"
 
-export SETUP_GVM_VERSION
+ONLY_DOCS=$(changeset_applies "$DOCS_CHANGESET")
+PACKAGING_CHANGES=$(changeset_applies "$PACKAGING_CHANGESET")
+GO_MOD_CHANGES=$(changeset_applies "^go.mod")
+
+export REPO
 export WORKSPACE
 export BIN
 export HW_TYPE
 export PLATFORM_TYPE
-export REPO
 export TMP_FOLDER
+export SNAPSHOT
+export PYTEST_ADDOPTS
+export OSS_MODULE_PATTERN
+export XPACK_MODULE_PATTERN
+
+export SETUP_GVM_VERSION
+export ASDF_MAGE_VERSION
+export SETUP_WIN_PYTHON_VERSION
+
+export DOCKER_COMPOSE_VERSION
 export DOCKER_REGISTRY
+
+export ONLY_DOCS
+export PACKAGING_CHANGES
+export GO_MOD_CHANGES

.buildkite/env-scripts/util.sh

@@ -9,11 +9,11 @@ add_bin_path() {
 }
 
 with_go() {
-    local go_version="${GOLANG_VERSION}"
+    local go_version="${GO_VERSION}"
     echo "Setting up the Go environment..."
     create_bin
     check_platform_architecture
-    retry 5 curl -sL -o ${BIN}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE}-${arch_type}"
+    retry_with_count 5 curl -sL -o ${BIN}/gvm "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${PLATFORM_TYPE}-${arch_type}"
     export PATH="${PATH}:${BIN}"
     chmod +x ${BIN}/gvm
     eval "$(gvm "$go_version")"
@@ -60,7 +60,7 @@ check_platform_architecture() {
   esac
 }
 
-retry() {
+retry_with_count() {
     local retries=$1
     shift
     local count=0
@@ -89,16 +89,16 @@ are_files_changed() {
   fi
 }
 
-cleanup() {
-  echo "Deleting temporary files..."
-  rm -rf ${BIN}/${TMP_FOLDER}.*
-  echo "Done."
+changeset_applies() {
+  local changeset=$1
+  if are_files_changed "$changeset"; then
+    echo true
+  else
+    echo false
+  fi
 }
 
-unset_secrets () {
-  for var in $(printenv | sed 's;=.*;;' | sort); do
-    if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then
-      unset "$var"
-    fi
-  done
+set_git_config() {
+  git config user.name "${GITHUB_USERNAME_SECRET}"
+  git config user.email "${GITHUB_EMAIL_SECRET}"
 }