Merge remote-tracking branch 'upstream/main' into beatsauth

Author: khushijain21

.buildkite/bk.integration-fips.pipeline.yml

@@ -4,9 +4,9 @@ env:
   ASDF_MAGE_VERSION: 1.14.0
   MS_GOTOOLCHAIN_TELEMETRY_ENABLED: "0"
 
-  IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1757120457"
-  IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1757120457"
-  IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1757120457"
+  IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1757725254"
+  IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1757725254"
+  IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1757725254"
   ASDF_TERRAFORM_VERSION: 1.9.2
 
 # This section is used to define the plugins that will be used in the pipeline.

.buildkite/bk.integration.pipeline.yml

@@ -6,12 +6,12 @@ env:
 
   # The following images are defined here and their values will be updated by updatecli
   # Please do not change them manually.
-  IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1757120457"
-  IMAGE_UBUNTU_2404_ARM_64: "platform-ingest-elastic-agent-ubuntu-2404-aarch64-1757120457"
-  IMAGE_RHEL_8: "platform-ingest-elastic-agent-rhel-8-1757120457"
-  IMAGE_DEBIAN_12: "platform-ingest-elastic-agent-debian-12-1757120457"
-  IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1757120457"
-  IMAGE_WIN_2025: "platform-ingest-elastic-agent-windows-2025-1757120457"
+  IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1757725254"
+  IMAGE_UBUNTU_2404_ARM_64: "platform-ingest-elastic-agent-ubuntu-2404-aarch64-1757725254"
+  IMAGE_RHEL_8: "platform-ingest-elastic-agent-rhel-8-1757725254"
+  IMAGE_DEBIAN_12: "platform-ingest-elastic-agent-debian-12-1757725254"
+  IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1757725254"
+  IMAGE_WIN_2025: "platform-ingest-elastic-agent-windows-2025-1757725254"
   ASDF_TERRAFORM_VERSION: 1.9.2
 
 # This section is used to define the plugins that will be used in the pipeline.

.buildkite/integration.pipeline.yml

@@ -6,8 +6,8 @@ env:
 
   # The following images are defined here and their values will be updated by updatecli
   # Please do not change them manually.
-  IMAGE_UBUNTU_2204_X86_64: "platform-ingest-elastic-agent-ubuntu-2204-1757120457"
-  IMAGE_UBUNTU_2204_ARM_64: "platform-ingest-elastic-agent-ubuntu-2204-aarch64-1757120457"
+  IMAGE_UBUNTU_2204_X86_64: "platform-ingest-elastic-agent-ubuntu-2204-1757725254"
+  IMAGE_UBUNTU_2204_ARM_64: "platform-ingest-elastic-agent-ubuntu-2204-aarch64-1757725254"
 
 common:
   - vault_docker_login: &vault_docker_login

.buildkite/pipeline.yml

@@ -5,12 +5,12 @@ env:
 
   # The following images are defined here and their values will be updated by updatecli
   # Please do not change them manually.
-  IMAGE_UBUNTU_2204_X86_64: "platform-ingest-elastic-agent-ubuntu-2204-1757120457"
-  IMAGE_UBUNTU_2204_ARM_64: "platform-ingest-elastic-agent-ubuntu-2204-aarch64-1757120457"
-  IMAGE_WIN_2016: "platform-ingest-elastic-agent-windows-2016-1757120457"
-  IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1757120457"
-  IMAGE_WIN_10: "platform-ingest-elastic-agent-windows-10-1757120457"
-  IMAGE_WIN_11: "platform-ingest-elastic-agent-windows-11-1757120457"
+  IMAGE_UBUNTU_2204_X86_64: "platform-ingest-elastic-agent-ubuntu-2204-1757725254"
+  IMAGE_UBUNTU_2204_ARM_64: "platform-ingest-elastic-agent-ubuntu-2204-aarch64-1757725254"
+  IMAGE_WIN_2016: "platform-ingest-elastic-agent-windows-2016-1757725254"
+  IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1757725254"
+  IMAGE_WIN_10: "platform-ingest-elastic-agent-windows-10-1757725254"
+  IMAGE_WIN_11: "platform-ingest-elastic-agent-windows-11-1757725254"
 
 steps:
   - label: "check-ci"

NOTICE-fips.txt

@@ -108,6 +108,11 @@ To build a local version of the agent for development, run the command below. Th
 DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64 PACKAGES=tar.gz mage -v package
 ```
 
+If you build the same agent package often (when running integration tests, for example),
+you can also set KEEP_ARCHIVE=true in your environment. The packaging step will then
+avoid deleting the binary archive after the package is generated, removing the need to
+re-download binaries on every invocation.
+
 The resulting package will be produced in the build/distributions directory. The version is controlled by the value in [version.go](version/version.go).
 To install the agent extract the package and run the install command:
 

NOTICE.txt

@@ -31,13 +31,7 @@ type downloadRequest struct {
 // It writes to the destination file as it downloads it, without
 // loading the entire file into memory.
 func downloadFile(downloadRequest *downloadRequest) error {
-	targetFile, err := os.Create(downloadRequest.TargetPath)
-	if err != nil {
-		return fmt.Errorf("creating file: %w", err)
-	}
-	defer func() {
-		_ = targetFile.Close()
-	}()
+	stat, _ := os.Stat(downloadRequest.TargetPath)
 
 	exp := getExponentialBackoff(3)
 
@@ -47,6 +41,11 @@ func downloadFile(downloadRequest *downloadRequest) error {
 		if err != nil {
 			return fmt.Errorf("creating request: %w", err)
 		}
+		// if the target file already exists, add the If-Modified-Since header
+		if stat != nil {
+			req.Header.Add("If-Modified-Since", stat.ModTime().Format(http.TimeFormat))
+		}
+
 		resp, err := http.DefaultClient.Do(req)
 		if err != nil {
 			retryCount++
@@ -55,23 +54,36 @@ func downloadFile(downloadRequest *downloadRequest) error {
 		defer func() {
 			_ = resp.Body.Close()
 		}()
+
+		if resp.StatusCode == http.StatusNotModified {
+			return nil
+		}
+
+		targetFile, err := os.Create(downloadRequest.TargetPath)
+		if err != nil {
+			return fmt.Errorf("creating file: %w", err)
+		}
+		defer func() {
+			_ = targetFile.Close()
+		}()
+
 		_, err = io.Copy(targetFile, resp.Body)
 		if err != nil {
 			// try to drain the body before returning to ensure the connection can be reused
 			_, _ = io.Copy(io.Discard, resp.Body)
 			return fmt.Errorf("writing file %s: %w", targetFile.Name(), err)
 		}
 
+		_ = os.Chmod(targetFile.Name(), 0666)
+
 		return nil
 	}
 
-	err = backoff.Retry(download, exp)
+	err := backoff.Retry(download, exp)
 	if err != nil {
 		return err
 	}
 
-	_ = os.Chmod(targetFile.Name(), 0666)
-
 	return nil
 }
 

README.md

@@ -261,6 +261,7 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
 		testArgs = append(testArgs,
 			"-covermode=atomic",
 			"-coverprofile="+params.CoverageProfileFile,
+			"-coverpkg=./...", // needed to calculate the coverage across the elastic-agent packages
 		)
 	}