automate the ironbank docker context generation (#32251)

.ci/packaging.groovy

@@ -400,6 +400,7 @@ def release(type){
       dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
       dir("${env.BEATS_FOLDER}") {
         sh(label: "mage package ${type} ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
+        sh(label: "mage ironbank ${type} ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage ironbank')
         def folder = getBeatsName(env.BEATS_FOLDER)
         uploadPackagesToGoogleBucket(
           credentialsId: env.JOB_GCS_EXT_CREDENTIALS,

auditbeat/magefile.go

@@ -101,6 +101,14 @@ func Package() {
 	mg.SerialDeps(devtools.Package, TestPackages)
 }
 
+// Package packages the Beat for IronBank distribution.
+//
+// Use SNAPSHOT=true to build snapshots.
+func Ironbank() error {
+	fmt.Println(">> Ironbank: this module is not subscribed to the IronBank releases.")
+	return nil
+}
+
 // TestPackages tests the generated packages (i.e. file modes, owners, groups).
 func TestPackages() error {
 	return devtools.TestPackages()

dev-tools/mage/common.go

@@ -374,8 +374,9 @@ func unzip(sourceFile, destinationDir string) error {
 	return nil
 }
 
-// Tar compress a directory using tar + gzip algorithms
-func Tar(src string, targetFile string) error {
+// Tar compress a directory using tar + gzip algorithms but without adding
+// the directory
+func TarWithOptions(src string, targetFile string, trimSource bool) error {
 	fmt.Printf(">> creating TAR file from directory: %s, target: %s\n", src, targetFile)
 
 	f, err := os.Create(targetFile)
@@ -409,6 +410,15 @@ func Tar(src string, targetFile string) error {
 		// must provide real name
 		// (see https://golang.org/src/archive/tar/common.go?#L626)
 		header.Name = filepath.ToSlash(file)
+		// Replace the source folder in the files to be compressed
+		if trimSource {
+			header.Name = strings.ReplaceAll(filepath.ToSlash(file), filepath.ToSlash(src), "")
+			header.Name = strings.TrimPrefix(header.Name, "/")
+			if header.Name == "" {
+				fmt.Print(">> skipping root directory\n")
+				return nil
+			}
+		}
 
 		// write header
 		if err := tw.WriteHeader(header); err != nil {
@@ -441,6 +451,11 @@ func Tar(src string, targetFile string) error {
 	return nil
 }
 
+// Tar compress a directory using tar + gzip algorithms
+func Tar(src string, targetFile string) error {
+	return TarWithOptions(src, targetFile, false)
+}
+
 func untar(sourceFile, destinationDir string) error {
 	file, err := os.Open(sourceFile)
 	if err != nil {

dev-tools/mage/copy.go

@@ -33,6 +33,16 @@ func Copy(src, dest string) error {
 	return copy.Execute()
 }
 
+// Copy copies a file and preserves the permissions.
+func CopyFile(src, dest string) error {
+	copy := &CopyTask{Source: src, Dest: dest}
+	info, err := os.Stat(src)
+	if err != nil {
+		return errors.Wrapf(err, "copy failed: cannot stat source file %v", src)
+	}
+	return copy.fileCopy(src, dest, info)
+}
+
 // CopyTask copies a file or directory (recursively) and preserves the permissions.
 type CopyTask struct {
 	Source   string           // Source directory or file.

dev-tools/mage/pkg.go

@@ -24,6 +24,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strconv"
+	"strings"
 
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
@@ -115,6 +116,112 @@ func Package() error {
 	return nil
 }
 
+// Package packages the Beat for IronBank distribution.
+//
+// Use SNAPSHOT=true to build snapshots.
+func Ironbank() error {
+	if runtime.GOARCH != "amd64" {
+		fmt.Printf(">> IronBank images are only supported for amd64 arch (%s is not supported)\n", runtime.GOARCH)
+		return nil
+	}
+	if err := prepareIronbankBuild(); err != nil {
+		return fmt.Errorf("failed to prepare the IronBank context: %w", err)
+	}
+	if err := saveIronbank(); err != nil {
+		return fmt.Errorf("failed to save the IronBank context: %w", err)
+	}
+	return nil
+}
+
+func getIronbankContextName() string {
+	version, _ := BeatQualifiedVersion()
+	ironbankBinaryName := "{{.Name}}-ironbank-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}-docker-build-context"
+	// TODO: get the name of the project
+	outputDir, _ := Expand(ironbankBinaryName, map[string]interface{}{
+		"Name":    BeatName,
+		"Version": version,
+	})
+	return outputDir
+}
+
+func prepareIronbankBuild() error {
+	fmt.Println(">> prepareIronbankBuild: prepare the IronBank container context.")
+	buildDir := filepath.Join("build", getIronbankContextName())
+	beatsDir, err := ElasticBeatsDir()
+	if err != nil {
+		return fmt.Errorf("could not get the base dir: %w", err)
+	}
+
+	templatesDir := filepath.Join(beatsDir, "dev-tools", "packaging", "templates", "ironbank", BeatName)
+
+	data := map[string]interface{}{
+		"MajorMinor": BeatMajorMinorVersion(),
+	}
+
+	err = filepath.Walk(templatesDir, func(path string, info os.FileInfo, _ error) error {
+		if !info.IsDir() {
+			target := strings.TrimSuffix(
+				filepath.Join(buildDir, filepath.Base(path)),
+				".tmpl",
+			)
+
+			err := ExpandFile(path, target, data)
+			if err != nil {
+				return fmt.Errorf("expanding template '%s' to '%s': %w", path, target, err)
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("cannot create templates for the IronBank: %w", err)
+	}
+
+	// copy license
+	sourceLicense := filepath.Join(beatsDir, "dev-tools", "packaging", "files", "ironbank", "LICENSE")
+	targetLicense := filepath.Join(buildDir, "LICENSE")
+	if err := CopyFile(sourceLicense, targetLicense); err != nil {
+		return fmt.Errorf("cannot copy LICENSE file for the IronBank: %w", err)
+	}
+
+	// copy specific files for the given beat
+	sourceBeatPath := filepath.Join(beatsDir, "dev-tools", "packaging", "files", "ironbank", BeatName)
+	if _, err := os.Stat(sourceBeatPath); !os.IsNotExist(err) {
+		if err := Copy(sourceBeatPath, buildDir); err != nil {
+			return fmt.Errorf("cannot create files for the IronBank: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func saveIronbank() error {
+	fmt.Println(">> saveIronbank: save the IronBank container context.")
+
+	ironbank := getIronbankContextName()
+	buildDir := filepath.Join("build", ironbank)
+	if _, err := os.Stat(buildDir); os.IsNotExist(err) {
+		return fmt.Errorf("cannot find the folder with the ironbank context: %+v", err)
+	}
+
+	distributionsDir := "build/distributions"
+	if _, err := os.Stat(distributionsDir); os.IsNotExist(err) {
+		err := os.MkdirAll(distributionsDir, 0750)
+		if err != nil {
+			return fmt.Errorf("cannot create folder for docker artifacts: %+v", err)
+		}
+	}
+	tarGzFile := filepath.Join(distributionsDir, ironbank+".tar.gz")
+
+	// Save the build context as tar.gz artifact
+	err := TarWithOptions(buildDir, tarGzFile, true)
+	if err != nil {
+		return fmt.Errorf("cannot compress the tar.gz file: %+v", err)
+	}
+
+	return errors.Wrap(CreateSHA512File(tarGzFile), "failed to create .sha512 file")
+}
+
 // updateWithDarwinUniversal checks if darwin/amd64 and darwin/arm64, are listed
 // if so, the universal binary was built, then we need to package it as well.
 func updateWithDarwinUniversal(platforms BuildPlatformList) BuildPlatformList {

dev-tools/mage/settings.go

@@ -360,6 +360,14 @@ func BeatQualifiedVersion() (string, error) {
 	return version + "-" + versionQualifier, nil
 }
 
+func BeatMajorMinorVersion() string {
+	if v, _ := BeatQualifiedVersion(); v != "" {
+		parts := strings.SplitN(v, ".", 3)
+		return parts[0] + "." + parts[1]
+	}
+	return ""
+}
+
 // BeatVersion returns the Beat's version. The value can be overridden by
 // setting BEAT_VERSION in the environment.
 func beatVersion() (string, error) {

dev-tools/make/mage.mk

@@ -38,7 +38,7 @@ help:
 
 .PHONY: release
 release: mage
-	mage package
+	mage package ironbank
 
 stop-environment: