diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 8b009c8b064..d046661de40 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -135,17 +135,9 @@ steps: - github_commit_status: context: "heartbeat: Win 2022 Unit Tests" + # Heartbeat has no modules - label: ":ubuntu: Heartbeat: Go Integration Tests" command: | - set -euo pipefail - echo "~~~ Installing @elastic/synthetics" - npm install -g @elastic/synthetics - - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet heartbeat - - echo "~~~ Running tests" cd heartbeat mage goIntegTest retry: @@ -162,14 +154,9 @@ steps: - github_commit_status: context: "heartbeat: Go Integration Tests" + # Heartbeat has no modules yet - label: ":ubuntu: Heartbeat: Python Integration Tests" command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet heartbeat - - echo "~~~ Running tests" cd heartbeat mage pythonIntegTest retry: diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index 340a68bf97a..57892523478 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -67,7 +67,7 @@ steps: - "libbeat/build/*.json" notify: - github_commit_status: - context: "libbeat: Ununtu x86_64 Unit Tests" + context: "libbeat: Ubuntu x86_64 Unit Tests" - label: ":ubuntu: Libbeat: Go Integration Tests" key: "mandatory-int-test" diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index 6cc305e28c4..86b08834278 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -22,6 +22,9 @@ env: K8S_VERSION: "v1.29.0" ASDF_KIND_VERSION: "0.20.0" + # Module Tests + BEAT_PATH: "metricbeat" + # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -80,14 +83,10 @@ steps: - github_commit_status: context: "metricbeat: Ubuntu x86_64 Unit Tests" - - label: ":ubuntu: Metricbeat: Go Integration Tests" + - label: ":ubuntu: Metricbeat: Go Integration Tests (Module)" key: "mandatory-int-test" command: | set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet metricbeat - echo "~~~ Running tests" export KUBECONFIG="$$PWD/kubecfg" cd metricbeat @@ -104,16 +103,12 @@ steps: - "metricbeat/build/*.json" notify: - github_commit_status: - context: "metricbeat: Go Integration Tests" + context: "metricbeat: Go Integration Tests (Module)" - - label: ":ubuntu: Metricbeat: Python Integration Tests" + - label: ":ubuntu: Metricbeat: Python Integration Tests (Module)" key: "mandatory-python-int-test" command: | set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet metricbeat - echo "~~~ Running tests" export KUBECONFIG="$$PWD/kubecfg" cd metricbeat diff --git a/.buildkite/scripts/changesets.psm1 b/.buildkite/scripts/changesets.psm1 deleted file mode 100644 index 10e4d31a8b6..00000000000 --- a/.buildkite/scripts/changesets.psm1 +++ /dev/null @@ -1,64 +0,0 @@ -function ArePathsChanged($patterns) { - $changedlist = @() - foreach ($pattern in $patterns) { - $changedFiles = & git diff --name-only "HEAD@{1}" HEAD | Select-String -Pattern $pattern -SimpleMatch - if ($changedFiles) { - $changedlist += $changedFiles - } - } - if ($changedlist) { - Write-Host "--- Files changed: $changedlist" - return $true - } - else { - Write-Host "--- No files changed within specified changeset: $patterns" - return $false - } -} - -function AreChangedOnlyPaths($patterns) { - $changedFiles = & git diff --name-only "HEAD@{1}" HEAD - Write-Host "--- Git Diff result:" - Write-Host "$changedFiles" - - $matchedFiles = @() - foreach ($pattern in $patterns) { - $matched = $changedFiles | Select-String -Pattern $pattern -SimpleMatch - if ($matched) { - $matchedFiles += $matched - } - } - if (($matchedFiles.Count -eq $changedFiles.Count) -or ($changedFiles.Count -eq 0)) { - return $true - } - return $false -} - -# This function sets a `MODULE` env var, required by IT tests, containing a comma separated list of modules for a given beats project (specified via the first argument). -# The list is built depending on directories that have changed under `modules/` excluding anything else such as asciidoc and png files. -# `MODULE` will empty if no changes apply. -function DefineModuleFromTheChangeSet($projectPath) { - $projectPathTransformed = $projectPath -replace '/', '\\' - $projectPathExclusion = "((?!^$projectPathTransformed\\\/).)*\$" - $exclude = @("^($projectPathExclusion|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") - - $changedModules = '' - - $moduleDirs = Get-ChildItem -Directory "$projectPath\module" - foreach($moduleDir in $moduleDirs) { - if((ArePathsChanged($moduleDir)) -and !(AreChangedOnlyPaths($exclude))) { - if(!$changedModules) { - $changedModules = $moduleDir.Name - } - else { - $changedModules += ',' + $moduleDir.Name - } - } - } - - if ($changedModules) { - $env:MODULE = $changedModules - Write-Output "~~~ Set env var MODULE to [$env:MODULE]" - Write-Output "~~~ Resuming commands" - } -} diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index 7f5eec0cb8b..40ef8c65aef 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -26,6 +26,9 @@ env: RACE_DETECTOR: "true" TEST_COVERAGE: "true" + # Module tests + BEAT_PATH: "x-pack/auditbeat" + steps: - group: "Check/Update" key: "x-pack-auditbeat-check-update" @@ -59,14 +62,9 @@ steps: key: "x-pack-auditbeat-mandatory-tests" steps: - - label: ":ubuntu: x-pack/auditbeat: Build Tests" + - label: ":ubuntu: x-pack/auditbeat: Build Tests (Module)" key: "mandatory-linux-unit-test" command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/auditbeat - echo "~~~ Running tests" cd x-pack/auditbeat mage update build test retry: @@ -81,7 +79,7 @@ steps: - "x-pack/auditbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/auditbeat: Build Tests" + context: "x-pack/auditbeat: Build Tests (Module)" - label: ":rhel: x-pack/auditbeat: RHEL9 Unit Tests" key: "mandatory-rhel9-unit-test" diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml index 14f94df67f8..72e5d8e24b9 100644 --- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -68,14 +68,10 @@ steps: - github_commit_status: context: "x-pack/dockerlogbeat: Ubuntu x86_64 Unit Tests" + # x-pack/dockerlogbeat has no modules yet - label: ":ubuntu: x-pack/dockerlogbeat: Go Integration Tests" key: "mandatory-int-test" command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/dockerlogbeat - echo "~~~ Running tests" cd x-pack/dockerlogbeat mage goIntegTest retry: diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index fde5db99bb9..b18b21ecde3 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -25,6 +25,9 @@ env: RACE_DETECTOR: "true" TEST_COVERAGE: "true" + # Module tests + BEAT_PATH: "x-pack/metricbeat" + steps: - group: "Check/Update" key: "x-pack-metricbeat-check-update" @@ -76,17 +79,13 @@ steps: - github_commit_status: context: "x-pack/metricbeat: Ubuntu x86_64 Unit Tests" - - label: ":ubuntu: x-pack/metricbeat: Go (MODULE) Integration Tests" + - label: ":ubuntu: x-pack/metricbeat: Go Integration Tests (Module)" key: "mandatory-int-test" env: TEST_TAGS: "oracle" command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/metricbeat - echo "~~~ Running tests" - cd x-pack/metricbeat && mage goIntegTest + cd x-pack/metricbeat + mage goIntegTest retry: automatic: - limit: 1 @@ -99,19 +98,15 @@ steps: - "x-pack/metricbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/metricbeat: Go (MODULE) Integration Tests" + context: "x-pack/metricbeat: Go Integration Tests (Module)" - - label: ":ubuntu: x-pack/metricbeat: Python (MODULE) Integration Tests" + - label: ":ubuntu: x-pack/metricbeat: Python Integration Tests (Module)" key: "mandatory-python-int-test" env: TEST_TAGS: "oracle" command: | - set -euo pipefail - # defines the MODULE env var based on what's changed in a PR - source .buildkite/scripts/changesets.sh - defineModuleFromTheChangeSet x-pack/metricbeat - echo "~~~ Running tests" - cd x-pack/metricbeat && mage pythonIntegTest + cd x-pack/metricbeat + mage pythonIntegTest retry: automatic: - limit: 1 @@ -124,7 +119,7 @@ steps: - "x-pack/metricbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/metricbeat: Python (MODULE) Integration Tests" + context: "x-pack/metricbeat: Python Integration Tests (Module)" - label: ":windows: x-pack/metricbeat: Win 2016 Unit Tests" command: | diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index d3c7b8cb476..b77198a32c6 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -50,12 +50,9 @@ steps: key: "x-pack-winlogbeat-mandatory-tests" steps: - - label: ":windows: x-pack/winlogbeat Win 2019 Unit (MODULE) Tests" - key: "mandatory-win-2019-module-unit-tests" + - label: ":windows: x-pack/winlogbeat Win 2019 Unit Tests" + key: "mandatory-win-2019-unit-tests" command: | - Import-Module ./.buildkite/scripts/changesets.psm1 - defineModuleFromTheChangeSet 'x-pack/winlogbeat' - Write-Output "~~~ Running tests" Set-Location -Path x-pack/winlogbeat mage build unitTest retry: @@ -72,7 +69,7 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/winlogbeat Win 2019 Unit (MODULE) Tests" + context: "x-pack/winlogbeat Win 2019 Unit Tests" - label: ":windows: x-pack/winlogbeat: Win 2016 Unit Tests" command: | diff --git a/dev-tools/mage/module_changeset.go b/dev-tools/mage/module_changeset.go new file mode 100644 index 00000000000..c7ffdbfd9d9 --- /dev/null +++ b/dev-tools/mage/module_changeset.go @@ -0,0 +1,164 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mage + +import ( + "fmt" + "log" + "os" + "regexp" + "strings" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" +) + +// DefineModules checks which modules were changed and populates MODULE environment variable, +// so that CI would run tests only for the changed ones. +// If no modules were changed MODULE variable won't be defined. +func DefineModules() { + // If MODULE is set in Buildkite pipeline step, skip variable further definition + if os.Getenv("MODULE") != "" { + return + } + + if mg.Verbose() { + fmt.Printf("Detecting changes in modules\n") + } + + beatPath := os.Getenv("BEAT_PATH") + if beatPath == "" { + log.Fatal("BEAT_PATH is not defined") + } + + var modulePattern = fmt.Sprintf("^%s\\/module\\/([^\\/]+)\\/.*", beatPath) + + moduleRegex, err := regexp.Compile(modulePattern) + if err != nil { + log.Fatal("failed to compile regex: " + err.Error()) + } + + modules := map[string]struct{}{} + for _, line := range getDiff() { + if !shouldIgnore(line) { + matches := moduleRegex.FindStringSubmatch(line) + if len(matches) > 0 { + modules[matches[1]] = struct{}{} + } + } + } + + keys := make([]string, len(modules)) + i := 0 + for k := range modules { + keys[i] = k + i++ + } + + moduleVar := strings.Join(keys, ",") + + if moduleVar != "" { + err = os.Setenv("MODULE", moduleVar) + if err != nil { + return + } + + log.Printf("Detected changes in module(s): %s\n", moduleVar) + } else { + log.Printf("No changed modules found") + } +} + +func shouldIgnore(file string) bool { + ignoreList := []string{".asciidoc", ".png"} + for ext := range ignoreList { + if strings.HasSuffix(file, ignoreList[ext]) { + return true + } + } + return false +} + +func getDiff() []string { + commitRange := getCommitRange() + var output, err = sh.Output("git", "diff", "--name-only", commitRange) + if err != nil { + log.Fatal("git Diff failed: %w", err) + } + + printWhenVerbose("Git Diff result: %s\n", output) + + return strings.Split(output, "\n") +} + +func getFromCommit() string { + if baseBranch := os.Getenv("BUILDKITE_PULL_REQUEST_BASE_BRANCH"); baseBranch != "" { + printWhenVerbose("PR branch: %s\n", baseBranch) + + return getBranchName(baseBranch) + } + + if previousCommit := getPreviousCommit(); previousCommit != "" { + printWhenVerbose("Git from commit: %s\n", previousCommit) + + return previousCommit + } else { + commit, err := getBuildkiteCommit() + if err != nil { + log.Fatal(err) + } + printWhenVerbose("Git from commit: %s\n", commit) + + return commit + } +} + +func getPreviousCommit() string { + var output, _ = sh.Output("git", "rev-parse", "HEAD^") + printWhenVerbose("Git previous commit: %s\n", output) + + return strings.TrimSpace(output) +} + +func getCommitRange() string { + commit, err := getBuildkiteCommit() + if err != nil { + log.Fatal(err) + } + + return fmt.Sprintf("%s...%s", getFromCommit(), commit) +} + +func getBranchName(branch string) string { + return fmt.Sprintf("origin/%s", branch) +} + +func printWhenVerbose(template string, parameter string) { + if mg.Verbose() { + fmt.Printf(template, parameter) + } +} + +func getBuildkiteCommit() (string, error) { + commit := os.Getenv("BUILDKITE_COMMIT") + if commit == "" { + log.Fatal("BUILDKITE_COMMIT is not defined") + } + + return commit, nil +} diff --git a/metricbeat/magefile.go b/metricbeat/magefile.go index c3647359b29..7bf9f7de48b 100644 --- a/metricbeat/magefile.go +++ b/metricbeat/magefile.go @@ -214,6 +214,10 @@ func IntegTest() { // Use TEST_TAGS=tag1,tag2 to add additional build tags. // Use MODULE=module to run only tests for `module`. func GoIntegTest(ctx context.Context) error { + if os.Getenv("CI") == "true" { + mg.Deps(devtools.DefineModules) + } + if !devtools.IsInIntegTestEnv() { mg.SerialDeps(Fields, Dashboards) } @@ -226,6 +230,10 @@ func GoIntegTest(ctx context.Context) error { // Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. // Use any other PYTEST_* environment variable to influence the behavior of pytest. func PythonIntegTest(ctx context.Context) error { + if os.Getenv("CI") == "true" { + mg.Deps(devtools.DefineModules) + } + if !devtools.IsInIntegTestEnv() { mg.SerialDeps(Fields, Dashboards) } diff --git a/x-pack/auditbeat/magefile.go b/x-pack/auditbeat/magefile.go index 8ffcbf84c89..45b34095481 100644 --- a/x-pack/auditbeat/magefile.go +++ b/x-pack/auditbeat/magefile.go @@ -8,8 +8,11 @@ package main import ( "fmt" + "os" "time" + "github.com/elastic/beats/v7/dev-tools/mage/target/test" + "github.com/magefile/mage/mg" "go.uber.org/multierr" @@ -25,8 +28,6 @@ import ( _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" //mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest/docker" - //mage:import - _ "github.com/elastic/beats/v7/dev-tools/mage/target/test" ) func init() { @@ -151,3 +152,12 @@ func ExportDashboard() error { func Dashboards() error { return devtools.KibanaDashboards(devtools.OSSBeatDir("module"), "module") } + +// Test runs all available tests (unitTest + integTest) +func Test() { + if os.Getenv("CI") == "true" { + mg.Deps(devtools.DefineModules) + } + + test.Test() +} diff --git a/x-pack/metricbeat/magefile.go b/x-pack/metricbeat/magefile.go index dfcc1a794c5..1de1a064d1f 100644 --- a/x-pack/metricbeat/magefile.go +++ b/x-pack/metricbeat/magefile.go @@ -234,6 +234,10 @@ func IntegTest() { // Use TEST_TAGS=tag1,tag2 to add additional build tags. // Use MODULE=module to run only tests for `module`. func GoIntegTest(ctx context.Context) error { + if os.Getenv("CI") == "true" { + mg.Deps(devtools.DefineModules) + } + if !devtools.IsInIntegTestEnv() { mg.SerialDeps(Fields, Dashboards) } @@ -246,6 +250,10 @@ func GoIntegTest(ctx context.Context) error { // Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. // Use any other PYTEST_* environment variable to influence the behavior of pytest. func PythonIntegTest(ctx context.Context) error { + if os.Getenv("CI") == "true" { + mg.Deps(devtools.DefineModules) + } + if !devtools.IsInIntegTestEnv() { mg.SerialDeps(Fields, Dashboards) }