From 1103ee3aedaa6efba43716aa7955c583c9d7e146 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 15 Apr 2024 12:45:02 +0200 Subject: [PATCH 1/7] Remove created agents (docker-custom-agent and k8s) --- internal/testrunner/runners/system/runner.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index cbab1c4bd..3e42b4a2e 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -934,13 +934,15 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf logger.Debugf("Selected enrolled agent %q", agent.ID) r.unenrollAgentHandler = func(ctx context.Context) error { - if !r.options.RunIndependentElasticAgent { + // When not using independent agents, service deployers like kubernetes or custom agents create new Elastic Agent + createdNewAgent := svcInfo.Agent.Host.NamePrefix == "docker-custom-agent" || svcInfo.Agent.Host.NamePrefix == "kind-control-plane" + if !r.options.RunIndependentElasticAgent && !createdNewAgent { return nil } - logger.Debug("unenrolling agent...") + logger.Debug("removing agent...") err := r.options.KibanaClient.RemoveAgent(ctx, agent) if err != nil { - return fmt.Errorf("failed to unenroll agent %q: %w", agent.ID, err) + return fmt.Errorf("failed to remove agent %q: %w", agent.ID, err) } return nil } From 93111812d72675d68423b6d5ac926bbed678c229 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 15 Apr 2024 12:45:28 +0200 Subject: [PATCH 2/7] Rename handler --- internal/testrunner/runners/system/runner.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 3e42b4a2e..8d4cedcd7 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -111,7 +111,7 @@ type runner struct { serviceStateFilePath string // Execution order of following handlers is defined in runner.TearDown() method. - unenrollAgentHandler func(context.Context) error + removeAgentHandler func(context.Context) error deleteTestPolicyHandler func(context.Context) error deletePackageHandler func(context.Context) error resetAgentPolicyHandler func(context.Context) error @@ -374,11 +374,11 @@ func (r *runner) tearDownTest(ctx context.Context) error { r.resetAgentLogLevelHandler = nil } - if r.unenrollAgentHandler != nil { - if err := r.unenrollAgentHandler(cleanupCtx); err != nil { + if r.removeAgentHandler != nil { + if err := r.removeAgentHandler(cleanupCtx); err != nil { return err } - r.unenrollAgentHandler = nil + r.removeAgentHandler = nil } if r.deleteTestPolicyHandler != nil { @@ -933,7 +933,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf agent := agents[0] logger.Debugf("Selected enrolled agent %q", agent.ID) - r.unenrollAgentHandler = func(ctx context.Context) error { + r.removeAgentHandler = func(ctx context.Context) error { // When not using independent agents, service deployers like kubernetes or custom agents create new Elastic Agent createdNewAgent := svcInfo.Agent.Host.NamePrefix == "docker-custom-agent" || svcInfo.Agent.Host.NamePrefix == "kind-control-plane" if !r.options.RunIndependentElasticAgent && !createdNewAgent { From 27bd2a4d44af69ae35071611cfed1ae638e69eea Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 15 Apr 2024 12:51:09 +0200 Subject: [PATCH 3/7] Remove Elastic Agent in k8s --- internal/servicedeployer/kubernetes.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/servicedeployer/kubernetes.go b/internal/servicedeployer/kubernetes.go index 537836e09..f2976cb3c 100644 --- a/internal/servicedeployer/kubernetes.go +++ b/internal/servicedeployer/kubernetes.go @@ -49,7 +49,9 @@ type KubernetesServiceDeployerOptions struct { } type kubernetesDeployedService struct { - svcInfo ServiceInfo + svcInfo ServiceInfo + stackVersion string + profile *profile.Profile definitionsDir string } @@ -57,6 +59,7 @@ type kubernetesDeployedService struct { func (s kubernetesDeployedService) TearDown(ctx context.Context) error { logger.Debugf("uninstall custom Kubernetes definitions (directory: %s)", s.definitionsDir) + // Remove service definitionPaths, err := findKubernetesDefinitions(s.definitionsDir) if err != nil { return fmt.Errorf("can't find Kubernetes definitions in given directory (path: %s): %w", s.definitionsDir, err) @@ -71,6 +74,17 @@ func (s kubernetesDeployedService) TearDown(ctx context.Context) error { if err != nil { return fmt.Errorf("can't uninstall Kubernetes resources (path: %s): %w", s.definitionsDir, err) } + + // Remove Elastic Agent + elasticAgentManagedYaml, err := getElasticAgentYAML(s.profile, s.stackVersion) + if err != nil { + return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) + } + err = kubectl.DeleteStdin(ctx, elasticAgentManagedYaml) + if err != nil { + return fmt.Errorf("can't uninstall Elastic Agent Kubernetes resources (path: %s): %w", s.definitionsDir, err) + } + return nil } @@ -147,6 +161,8 @@ func (ksd KubernetesServiceDeployer) SetUp(ctx context.Context, svcInfo ServiceI return &kubernetesDeployedService{ svcInfo: svcInfo, definitionsDir: ksd.definitionsDir, + stackVersion: ksd.stackVersion, + profile: ksd.profile, }, nil } From 0b594e6817d051154a3a93dacb5977f43e4ca3aa Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 15 Apr 2024 16:06:01 +0200 Subject: [PATCH 4/7] Remove first Elastic Agent in the tear-down process (k8s) --- internal/servicedeployer/kubernetes.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/internal/servicedeployer/kubernetes.go b/internal/servicedeployer/kubernetes.go index f2976cb3c..c898eea9b 100644 --- a/internal/servicedeployer/kubernetes.go +++ b/internal/servicedeployer/kubernetes.go @@ -57,9 +57,17 @@ type kubernetesDeployedService struct { } func (s kubernetesDeployedService) TearDown(ctx context.Context) error { - logger.Debugf("uninstall custom Kubernetes definitions (directory: %s)", s.definitionsDir) + logger.Debug("uninstall Elastic Agent Kubernetes") + elasticAgentManagedYaml, err := getElasticAgentYAML(s.profile, s.stackVersion) + if err != nil { + return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) + } + err = kubectl.DeleteStdin(ctx, elasticAgentManagedYaml) + if err != nil { + return fmt.Errorf("can't uninstall Elastic Agent Kubernetes resources (path: %s): %w", s.definitionsDir, err) + } - // Remove service + logger.Debugf("uninstall custom Kubernetes definitions (directory: %s)", s.definitionsDir) definitionPaths, err := findKubernetesDefinitions(s.definitionsDir) if err != nil { return fmt.Errorf("can't find Kubernetes definitions in given directory (path: %s): %w", s.definitionsDir, err) @@ -75,16 +83,6 @@ func (s kubernetesDeployedService) TearDown(ctx context.Context) error { return fmt.Errorf("can't uninstall Kubernetes resources (path: %s): %w", s.definitionsDir, err) } - // Remove Elastic Agent - elasticAgentManagedYaml, err := getElasticAgentYAML(s.profile, s.stackVersion) - if err != nil { - return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) - } - err = kubectl.DeleteStdin(ctx, elasticAgentManagedYaml) - if err != nil { - return fmt.Errorf("can't uninstall Elastic Agent Kubernetes resources (path: %s): %w", s.definitionsDir, err) - } - return nil } From ddcc2f78f6c63c31eb653f9b854d2aa988566fd7 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 15 Apr 2024 16:44:42 +0200 Subject: [PATCH 5/7] Do not uninstall Elastic Agent if independent agents are set --- internal/servicedeployer/kubernetes.go | 29 +++++++++++++++----------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/internal/servicedeployer/kubernetes.go b/internal/servicedeployer/kubernetes.go index c898eea9b..3290a4aab 100644 --- a/internal/servicedeployer/kubernetes.go +++ b/internal/servicedeployer/kubernetes.go @@ -53,18 +53,22 @@ type kubernetesDeployedService struct { stackVersion string profile *profile.Profile + deployIndependentAgent bool + definitionsDir string } func (s kubernetesDeployedService) TearDown(ctx context.Context) error { - logger.Debug("uninstall Elastic Agent Kubernetes") - elasticAgentManagedYaml, err := getElasticAgentYAML(s.profile, s.stackVersion) - if err != nil { - return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) - } - err = kubectl.DeleteStdin(ctx, elasticAgentManagedYaml) - if err != nil { - return fmt.Errorf("can't uninstall Elastic Agent Kubernetes resources (path: %s): %w", s.definitionsDir, err) + if !s.deployIndependentAgent { + logger.Debug("uninstall Elastic Agent Kubernetes") + elasticAgentManagedYaml, err := getElasticAgentYAML(s.profile, s.stackVersion) + if err != nil { + return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) + } + err = kubectl.DeleteStdin(ctx, elasticAgentManagedYaml) + if err != nil { + return fmt.Errorf("can't uninstall Elastic Agent Kubernetes resources (path: %s): %w", s.definitionsDir, err) + } } logger.Debugf("uninstall custom Kubernetes definitions (directory: %s)", s.definitionsDir) @@ -157,10 +161,11 @@ func (ksd KubernetesServiceDeployer) SetUp(ctx context.Context, svcInfo ServiceI // to deploy Agent Pod. Because of this, hostname inside pod will be equal to the name of the k8s host. svcInfo.Agent.Host.NamePrefix = "kind-control-plane" return &kubernetesDeployedService{ - svcInfo: svcInfo, - definitionsDir: ksd.definitionsDir, - stackVersion: ksd.stackVersion, - profile: ksd.profile, + svcInfo: svcInfo, + definitionsDir: ksd.definitionsDir, + stackVersion: ksd.stackVersion, + profile: ksd.profile, + deployIndependentAgent: ksd.deployIndependentAgent, }, nil } From da7f8b86460c9122c0785962ca6b0913b398d38f Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 16 Apr 2024 09:40:42 +0200 Subject: [PATCH 6/7] Update debug messages Co-authored-by: Jaime Soriano Pastor --- internal/servicedeployer/kubernetes.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/servicedeployer/kubernetes.go b/internal/servicedeployer/kubernetes.go index 3290a4aab..f078ade0c 100644 --- a/internal/servicedeployer/kubernetes.go +++ b/internal/servicedeployer/kubernetes.go @@ -60,7 +60,7 @@ type kubernetesDeployedService struct { func (s kubernetesDeployedService) TearDown(ctx context.Context) error { if !s.deployIndependentAgent { - logger.Debug("uninstall Elastic Agent Kubernetes") + logger.Debug("Uninstall Elastic Agent Kubernetes") elasticAgentManagedYaml, err := getElasticAgentYAML(s.profile, s.stackVersion) if err != nil { return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) @@ -71,7 +71,7 @@ func (s kubernetesDeployedService) TearDown(ctx context.Context) error { } } - logger.Debugf("uninstall custom Kubernetes definitions (directory: %s)", s.definitionsDir) + logger.Debugf("Uninstall custom Kubernetes definitions (directory: %s)", s.definitionsDir) definitionPaths, err := findKubernetesDefinitions(s.definitionsDir) if err != nil { return fmt.Errorf("can't find Kubernetes definitions in given directory (path: %s): %w", s.definitionsDir, err) From 096811fa2dcf0dcf45d649bac7b2f9ff664acb92 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 16 Apr 2024 11:55:21 +0200 Subject: [PATCH 7/7] Enroll custom and k8s agents using the created test Agent Policy directly --- internal/servicedeployer/factory.go | 1 + internal/servicedeployer/kubernetes.go | 17 +++++++++++------ internal/testrunner/runners/system/runner.go | 11 +++++------ 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/internal/servicedeployer/factory.go b/internal/servicedeployer/factory.go index 7472881e9..b70544181 100644 --- a/internal/servicedeployer/factory.go +++ b/internal/servicedeployer/factory.go @@ -63,6 +63,7 @@ func Factory(options FactoryOptions) (ServiceDeployer, error) { Profile: options.Profile, DefinitionsDir: serviceDeployerPath, StackVersion: options.StackVersion, + PolicyName: options.PolicyName, RunSetup: options.RunSetup, RunTestsOnly: options.RunTestsOnly, RunTearDown: options.RunTearDown, diff --git a/internal/servicedeployer/kubernetes.go b/internal/servicedeployer/kubernetes.go index f078ade0c..786c0e00d 100644 --- a/internal/servicedeployer/kubernetes.go +++ b/internal/servicedeployer/kubernetes.go @@ -28,6 +28,7 @@ type KubernetesServiceDeployer struct { profile *profile.Profile definitionsDir string stackVersion string + policyName string deployIndependentAgent bool @@ -40,6 +41,7 @@ type KubernetesServiceDeployerOptions struct { Profile *profile.Profile DefinitionsDir string StackVersion string + PolicyName string DeployIndependentAgent bool @@ -52,6 +54,7 @@ type kubernetesDeployedService struct { svcInfo ServiceInfo stackVersion string profile *profile.Profile + policyName string deployIndependentAgent bool @@ -61,7 +64,7 @@ type kubernetesDeployedService struct { func (s kubernetesDeployedService) TearDown(ctx context.Context) error { if !s.deployIndependentAgent { logger.Debug("Uninstall Elastic Agent Kubernetes") - elasticAgentManagedYaml, err := getElasticAgentYAML(s.profile, s.stackVersion) + elasticAgentManagedYaml, err := getElasticAgentYAML(s.profile, s.stackVersion, s.policyName) if err != nil { return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) } @@ -115,6 +118,7 @@ func NewKubernetesServiceDeployer(opts KubernetesServiceDeployerOptions) (*Kuber profile: opts.Profile, definitionsDir: opts.DefinitionsDir, stackVersion: opts.StackVersion, + policyName: opts.PolicyName, runSetup: opts.RunSetup, runTestsOnly: opts.RunTestsOnly, runTearDown: opts.RunTearDown, @@ -142,7 +146,7 @@ func (ksd KubernetesServiceDeployer) SetUp(ctx context.Context, svcInfo ServiceI if ksd.runTearDown || ksd.runTestsOnly || ksd.deployIndependentAgent { logger.Debug("Skip install Elastic Agent in cluster") } else { - err = installElasticAgentInCluster(ctx, ksd.profile, ksd.stackVersion) + err = installElasticAgentInCluster(ctx, ksd.profile, ksd.stackVersion, ksd.policyName) if err != nil { return nil, fmt.Errorf("can't install Elastic-Agent in the Kubernetes cluster: %w", err) } @@ -166,6 +170,7 @@ func (ksd KubernetesServiceDeployer) SetUp(ctx context.Context, svcInfo ServiceI stackVersion: ksd.stackVersion, profile: ksd.profile, deployIndependentAgent: ksd.deployIndependentAgent, + policyName: ksd.policyName, }, nil } @@ -202,10 +207,10 @@ func findKubernetesDefinitions(definitionsDir string) ([]string, error) { return definitionPaths, nil } -func installElasticAgentInCluster(ctx context.Context, profile *profile.Profile, stackVersion string) error { +func installElasticAgentInCluster(ctx context.Context, profile *profile.Profile, stackVersion, policyName string) error { logger.Debug("install Elastic Agent in the Kubernetes cluster") - elasticAgentManagedYaml, err := getElasticAgentYAML(profile, stackVersion) + elasticAgentManagedYaml, err := getElasticAgentYAML(profile, stackVersion, policyName) if err != nil { return fmt.Errorf("can't retrieve Kubernetes file for Elastic Agent: %w", err) } @@ -220,7 +225,7 @@ func installElasticAgentInCluster(ctx context.Context, profile *profile.Profile, //go:embed _static/elastic-agent-managed.yaml.tmpl var elasticAgentManagedYamlTmpl string -func getElasticAgentYAML(profile *profile.Profile, stackVersion string) ([]byte, error) { +func getElasticAgentYAML(profile *profile.Profile, stackVersion, policyName string) ([]byte, error) { logger.Debugf("Prepare YAML definition for Elastic Agent running in stack v%s", stackVersion) appConfig, err := install.Configuration() @@ -241,7 +246,7 @@ func getElasticAgentYAML(profile *profile.Profile, stackVersion string) ([]byte, "kibanaURL": "https://kibana:5601", "caCertPem": caCert, "elasticAgentImage": appConfig.StackImageRefs(stackVersion).ElasticAgent, - "elasticAgentTokenPolicyName": getTokenPolicyName(stackVersion, defaulFleetTokenPolicyName), + "elasticAgentTokenPolicyName": getTokenPolicyName(stackVersion, policyName), }) if err != nil { return nil, fmt.Errorf("can't generate elastic agent manifest: %w", err) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 8d4cedcd7..1c5a0813d 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -805,7 +805,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf scenario.enrollingTime = enrollingTime scenario.agent = agentDeployed - service, svcInfo, err := r.setupService(ctx, config, serviceOptions, svcInfo, agentInfo, agentDeployed, serviceStateData) + service, svcInfo, err := r.setupService(ctx, config, serviceOptions, svcInfo, agentInfo, agentDeployed, policy, serviceStateData) if errors.Is(err, os.ErrNotExist) { logger.Debugf("No service deployer defined for this test") } else if err != nil { @@ -1096,7 +1096,7 @@ func (r *runner) prepareScenario(ctx context.Context, config *testConfig, svcInf return &scenario, nil } -func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOptions servicedeployer.FactoryOptions, svcInfo servicedeployer.ServiceInfo, agentInfo agentdeployer.AgentInfo, agentDeployed agentdeployer.DeployedAgent, state ServiceState) (servicedeployer.DeployedService, servicedeployer.ServiceInfo, error) { +func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOptions servicedeployer.FactoryOptions, svcInfo servicedeployer.ServiceInfo, agentInfo agentdeployer.AgentInfo, agentDeployed agentdeployer.DeployedAgent, policy *kibana.Policy, state ServiceState) (servicedeployer.DeployedService, servicedeployer.ServiceInfo, error) { logger.Debug("setting up service...") if r.options.RunTearDown || r.options.RunTestsOnly { svcInfo.Test.RunID = state.ServiceRunID @@ -1114,10 +1114,9 @@ func (r *runner) setupService(ctx context.Context, config *testConfig, serviceOp svcInfo.Logs.Folder.Local = agentInfo.Logs.Folder.Local } - // In case of custom agent (servicedeployer) enabling independent agents, update serviceOptions to include test policy too - if r.options.RunIndependentElasticAgent { - serviceOptions.PolicyName = agentInfo.Policy.Name - } + // In case of custom or kubernetes agents (servicedeployer) it is needed also the Agent Policy created + // for each test execution + serviceOptions.PolicyName = policy.Name if config.Service != "" { svcInfo.Name = config.Service