From e3348dfeaa7c17806582bcf569da7c54b7b4b822 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 17 Aug 2021 17:29:57 -0400 Subject: [PATCH] [Elastic Agent] Adjust installation path in container (#27396) (#27432) * Adjust atomic installer to extract in the install directory, removing the need to rename across filesystem boundaries. * Add comment. * Add changelog. * Fix install path on k8s. * Fix issue with paths. * Cleanup temps. * Add changelog. (cherry picked from commit 9ce228f0dab9059ab5706cfba7a5ff59231ec523) Co-authored-by: Blake Rouse --- x-pack/elastic-agent/CHANGELOG.next.asciidoc | 2 ++ .../pkg/agent/application/paths/common.go | 30 +++++++++++++++++++ x-pack/elastic-agent/pkg/agent/cmd/common.go | 2 ++ .../elastic-agent/pkg/agent/cmd/container.go | 12 ++++++-- .../elastic-agent/pkg/agent/cmd/enroll_cmd.go | 6 ++++ x-pack/elastic-agent/pkg/artifact/config.go | 6 ++-- .../pkg/artifact/download/fs/downloader.go | 4 +-- .../install/atomic/atomic_installer.go | 12 ++++++-- 8 files changed, 64 insertions(+), 10 deletions(-) diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index 4e0c6c10b8c..45fb3cfcc4d 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -79,6 +79,8 @@ - Fix apm-server supported outputs not being in sync with supported output types. {pull}26885[26885] - Set permissions during installation {pull}26665[26665] - Disable monitoring during fleet-server bootstrapping. {pull}27222[27222] +- Fix issue with atomic extract running in K8s {pull}27396[27396] +- Fix issue with install directory in state path in K8s {pull}27396[27396] ==== New features diff --git a/x-pack/elastic-agent/pkg/agent/application/paths/common.go b/x-pack/elastic-agent/pkg/agent/application/paths/common.go index b445cb0c40b..a9b80522f2d 100644 --- a/x-pack/elastic-agent/pkg/agent/application/paths/common.go +++ b/x-pack/elastic-agent/pkg/agent/application/paths/common.go @@ -28,6 +28,8 @@ var ( configPath string configFilePath string logsPath string + downloadsPath string + installPath string unversionedHome bool tmpCreator sync.Once ) @@ -44,6 +46,8 @@ func init() { fs.StringVar(&configPath, "path.config", configPath, "Config path is the directory Agent looks for its config file") fs.StringVar(&configFilePath, "c", DefaultConfigName, "Configuration file, relative to path.config") fs.StringVar(&logsPath, "path.logs", logsPath, "Logs path contains Agent log output") + fs.StringVar(&downloadsPath, "path.downloads", downloadsPath, "Downloads path contains binaries Agent downloads") + fs.StringVar(&installPath, "path.install", installPath, "Install path contains binaries Agent extracts") } // Top returns the top directory for Elastic Agent, all the versioned @@ -139,6 +143,32 @@ func VersionedHome(base string) string { return filepath.Join(base, "data", fmt.Sprintf("elastic-agent-%s", release.ShortCommit())) } +// Downloads returns the downloads directory for Agent +func Downloads() string { + if downloadsPath == "" { + return filepath.Join(Home(), "downloads") + } + return downloadsPath +} + +// SetDownloads updates the path for the downloads. +func SetDownloads(path string) { + downloadsPath = path +} + +// Install returns the install directory for Agent +func Install() string { + if installPath == "" { + return filepath.Join(Home(), "install") + } + return installPath +} + +// SetInstall updates the path for the install. +func SetInstall(path string) { + installPath = path +} + // initialTop returns the initial top-level path for the binary // // When nested in top-level/data/elastic-agent-${hash}/ the result is top-level/. diff --git a/x-pack/elastic-agent/pkg/agent/cmd/common.go b/x-pack/elastic-agent/pkg/agent/cmd/common.go index 3db6cff275d..e98c384cfaf 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/common.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/common.go @@ -44,6 +44,8 @@ func NewCommandWithArgs(args []string, streams *cli.IOStreams) *cobra.Command { cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.config")) cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("c")) cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.logs")) + cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.downloads")) + cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.install")) // logging flags cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("v")) diff --git a/x-pack/elastic-agent/pkg/agent/cmd/container.go b/x-pack/elastic-agent/pkg/agent/cmd/container.go index a8f9eab0f45..bce71fa874a 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/container.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/container.go @@ -334,6 +334,12 @@ func buildEnrollArgs(cfg setupConfig, token string, policyID string) ([]string, "--path.config", paths.Config(), "--path.logs", paths.Logs(), } + if paths.Downloads() != "" { + args = append(args, "--path.downloads", paths.Downloads()) + } + if paths.Install() != "" { + args = append(args, "--path.install", paths.Install()) + } if !paths.IsVersionHome() { args = append(args, "--path.home.unversioned") } @@ -704,16 +710,18 @@ func setPaths(statePath, configPath, logsPath string, writePaths bool) error { } } // sync the downloads to the data directory - srcDownloads := filepath.Join(paths.Home(), "downloads") destDownloads := filepath.Join(statePath, "data", "downloads") - if err := syncDir(srcDownloads, destDownloads); err != nil { + if err := syncDir(paths.Downloads(), destDownloads); err != nil { return fmt.Errorf("syncing download directory to STATE_PATH(%s) failed: %s", statePath, err) } + originalInstall := paths.Install() originalTop := paths.Top() paths.SetTop(topPath) paths.SetConfig(configPath) // when custom top path is provided the home directory is not versioned paths.SetVersionHome(false) + // install path stays on container default mount (otherwise a bind mounted directory could have noexec set) + paths.SetInstall(originalInstall) // set LOGS_PATH is given logsPath = envWithDefault(logsPath, "LOGS_PATH") if logsPath != "" { diff --git a/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd.go b/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd.go index 88b888ae63b..0acd4cba6c8 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd.go @@ -546,6 +546,12 @@ func (c *enrollCmd) startAgent(ctx context.Context) (<-chan *os.ProcessState, er "--path.home", paths.Top(), "--path.config", paths.Config(), "--path.logs", paths.Logs(), } + if paths.Downloads() != "" { + args = append(args, "--path.downloads", paths.Downloads()) + } + if paths.Install() != "" { + args = append(args, "--path.install", paths.Install()) + } if !paths.IsVersionHome() { args = append(args, "--path.home.unversioned") } diff --git a/x-pack/elastic-agent/pkg/artifact/config.go b/x-pack/elastic-agent/pkg/artifact/config.go index d1181b778d2..60eb1239ebd 100644 --- a/x-pack/elastic-agent/pkg/artifact/config.go +++ b/x-pack/elastic-agent/pkg/artifact/config.go @@ -5,7 +5,6 @@ package artifact import ( - "path/filepath" "runtime" "strings" "time" @@ -43,7 +42,6 @@ type Config struct { // DefaultConfig creates a config with pre-set default values. func DefaultConfig() *Config { - homePath := paths.Home() transport := httpcommon.DefaultHTTPTransportSettings() // binaries are a getting bit larger it might take >30s to download them @@ -51,8 +49,8 @@ func DefaultConfig() *Config { return &Config{ SourceURI: "https://artifacts.elastic.co/downloads/", - TargetDirectory: filepath.Join(homePath, "downloads"), - InstallPath: filepath.Join(homePath, "install"), + TargetDirectory: paths.Downloads(), + InstallPath: paths.Install(), HTTPTransportSettings: transport, } } diff --git a/x-pack/elastic-agent/pkg/artifact/download/fs/downloader.go b/x-pack/elastic-agent/pkg/artifact/download/fs/downloader.go index b1bac639c51..3d055e69e65 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/fs/downloader.go +++ b/x-pack/elastic-agent/pkg/artifact/download/fs/downloader.go @@ -116,13 +116,13 @@ func (e *Downloader) downloadFile(filename, fullPath string) (string, error) { func getDropPath(cfg *artifact.Config) string { // if drop path is not provided fallback to beats subfolder if cfg == nil || cfg.DropPath == "" { - return filepath.Join(paths.Home(), "downloads") + return paths.Downloads() } // if droppath does not exist fallback to beats subfolder stat, err := os.Stat(cfg.DropPath) if err != nil || !stat.IsDir() { - return filepath.Join(paths.Home(), "downloads") + return paths.Downloads() } return cfg.DropPath diff --git a/x-pack/elastic-agent/pkg/artifact/install/atomic/atomic_installer.go b/x-pack/elastic-agent/pkg/artifact/install/atomic/atomic_installer.go index a9406cf9815..0f84fdd78c5 100644 --- a/x-pack/elastic-agent/pkg/artifact/install/atomic/atomic_installer.go +++ b/x-pack/elastic-agent/pkg/artifact/install/atomic/atomic_installer.go @@ -13,7 +13,6 @@ import ( "github.com/hashicorp/go-multierror" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program" ) @@ -37,10 +36,19 @@ func NewInstaller(i embeddedInstaller) (*Installer, error) { // Install performs installation of program in a specific version. func (i *Installer) Install(ctx context.Context, spec program.Spec, version, installDir string) error { // tar installer uses Dir of installDir to determine location of unpack - tempDir, err := ioutil.TempDir(paths.TempDir(), "elastic-agent-install") + // + // installer is ran inside a tmp directory created in the parent installDir, this is so the atomic + // rename always occurs on the same mount path that holds the installation directory + tempDir, err := ioutil.TempDir(filepath.Dir(installDir), "tmp") if err != nil { return err } + + // always remove the entire tempDir + defer func() { + os.RemoveAll(tempDir) + }() + tempInstallDir := filepath.Join(tempDir, filepath.Base(installDir)) // cleanup install directory before Install