[9.0](backport #6907) Enhancement/6394 allow deb rpm to upgrade with endpoint tamper protection

changelog/fragments/1740166208-allow-deb-rpm-upgrade-with-tamper-protected-endpoint.yaml

@@ -0,0 +1,30 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: Allow upgrading deb or rpm agents when using Elastic Defend with tamper protection.
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/6907
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/6394

dev-tools/packaging/templates/linux/postinstall.sh.tmpl

@@ -6,13 +6,42 @@ commit_hash="{{ commit_short }}"
 version_dir="{{agent_package_version}}{{snapshot_suffix}}"
 symlink="/usr/share/elastic-agent/bin/elastic-agent"
 new_agent_dir="/var/lib/elastic-agent/data/elastic-agent-$version_dir-$commit_hash"
+new_endpoint_component_bin="$new_agent_dir/components/endpoint-security"
 
 # delete $symlink if exists
 if test -L "$symlink"; then
     echo "found symlink $symlink, unlink"
     unlink "$symlink"
 fi
 
+SERVICE_NAME="ElasticEndpoint"
+
+echo "Checking if $SERVICE_NAME is installed"
+if systemctl list-unit-files --type=service | grep -q "^${SERVICE_NAME}.service"; then
+
+    installed_endpoint_version="$(/opt/Elastic/Endpoint/elastic-endpoint version)"
+    archive_endpoint_version="$("$new_endpoint_component_bin" version)"
+
+    echo "${SERVICE_NAME} is installed at version ${installed_endpoint_version}"
+    if ! systemctl is-active --quiet "$SERVICE_NAME"; then
+        echo "$SERVICE_NAME is installed but not running"
+        if [[ "$installed_endpoint_version" == "$archive_endpoint_version" ]]; then
+            echo "New endpoint and installed endpoint versions are the same: \"${installed_endpoint_version}\""
+            echo "Starting $SERVICE_NAME"
+            sudo systemctl start ${SERVICE_NAME}
+        else
+            echo "New endpoint version  \"${archive_endpoint_version}\" is different than the one that's already
+            installed  \"${installed_endpoint_version}\""
+        fi
+    else
+        # Endpoint can already be running if tamper protection is not enabled
+        echo "$SERVICE_NAME is already running"
+    fi
+else
+    echo "$SERVICE_NAME is not installed"
+fi
+
+
 # create symlink to the new agent
 echo "create symlink "$symlink" to "$new_agent_dir/elastic-agent""
 ln -s "$new_agent_dir/elastic-agent" "$symlink"

dev-tools/packaging/templates/linux/preinstall.sh.tmpl

@@ -2,6 +2,33 @@
 
 set -e
 
+SERVICE_NAME="ElasticEndpoint"
+
+# Check if the endpoint service is installed
+if systemctl list-unit-files --type=service | grep -q "^${SERVICE_NAME}.service"; then
+    echo "$SERVICE_NAME is installed"
+
+    # Remove the vault directory if it exists
+    if [ -d "/opt/Elastic/Endpoint/state/vault" ]; then
+
+        # Check if the endpoint is running
+        if systemctl --quiet is-active $SERVICE_NAME; then
+            echo "$SERVICE_NAME is running, stopping it"
+            systemctl --quiet stop $SERVICE_NAME
+        else
+            echo "$SERVICE_NAME is not running"
+        fi
+
+        echo "$SERVICE_NAME is tamper protected"
+        echo "Removing $SERVICE_NAME vault"
+        rm -rf "/opt/Elastic/Endpoint/state/vault"
+    else
+        echo "$SERVICE_NAME tamper protection is not enabled"
+    fi
+else
+    echo "$SERVICE_NAME is not installed"
+fi
+
 commit_hash="{{ commit_short }}"
 version_dir="{{agent_package_version}}{{snapshot_suffix}}"
 symlink="/usr/share/elastic-agent/bin/elastic-agent"
@@ -12,27 +39,27 @@ old_agent_dir=""
 # upon upgrade we migrate the current symlink to an upgrade symlink as the previous
 # installed version will remove the symlink
 if test -L "$symlink"; then
-    resolved_symlink="$(readlink -- "$symlink")"
-    if ! [ -z "$resolved_symlink" ]; then
-        old_agent_dir="$( dirname "$resolved_symlink" )"
-        echo "previous installation directory $old_agent_dir"
-    else
-        echo "unable to read existing symlink"
-    fi
+	resolved_symlink="$(readlink -- "$symlink")"
+	if ! [ -z "$resolved_symlink" ]; then
+		old_agent_dir="$(dirname "$resolved_symlink")"
+		echo "previous installation directory $old_agent_dir"
+	else
+		echo "unable to read existing symlink"
+	fi
 
-    # copy the state files if there was a previous agent install
-    if ! [ -z "$old_agent_dir" ] && ! [ "$old_agent_dir" -ef "$new_agent_dir" ]; then
-        yml_path="$old_agent_dir/state.yml"
-        enc_path="$old_agent_dir/state.enc"
-        echo "migrate state from $old_agent_dir to $new_agent_dir"
+	# copy the state files if there was a previous agent install
+	if ! [ -z "$old_agent_dir" ] && ! [ "$old_agent_dir" -ef "$new_agent_dir" ]; then
+		yml_path="$old_agent_dir/state.yml"
+		enc_path="$old_agent_dir/state.enc"
+		echo "migrate state from $old_agent_dir to $new_agent_dir"
 
-        if test -f "$yml_path"; then
-            echo "found "$yml_path", copy to "$new_agent_dir"."
-            mkdir -p "$new_agent_dir"
-            cp "$yml_path" "$new_agent_dir"
-        else
-            echo "didn't find $yml_path"
-        fi
+		if test -f "$yml_path"; then
+			echo "found "$yml_path", copy to "$new_agent_dir"."
+			mkdir -p "$new_agent_dir"
+			cp "$yml_path" "$new_agent_dir"
+		else
+			echo "didn't find $yml_path"
+		fi
 
         if test -f "$enc_path"; then
             echo "found "$enc_path", copy to "$new_agent_dir"."
@@ -53,7 +80,7 @@ if test -L "$symlink"; then
         fi
     fi
 else
-    echo "no previous installation found"
+	echo "no previous installation found"
 
     # create dir in case it does not exist
     mkdir -p "$new_agent_dir"
@@ -71,4 +98,3 @@ else
         fi
     fi
 fi
-

pkg/testing/fixture.go

@@ -346,7 +346,6 @@ func (f *Fixture) RunBeat(ctx context.Context) error {
 		process.WithContext(ctx),
 		process.WithArgs(args),
 		process.WithCmdOptions(attachOutErr(stdOut, stdErr)))
-
 	if err != nil {
 		return fmt.Errorf("failed to spawn %s: %w", f.binaryName, err)
 	}
@@ -401,7 +400,8 @@ func RunProcess(t *testing.T,
 	lp Logger,
 	ctx context.Context, runLength time.Duration,
 	logOutput, allowErrs bool,
-	processPath string, args ...string) error {
+	processPath string, args ...string,
+) error {
 	if _, deadlineSet := ctx.Deadline(); !deadlineSet {
 		t.Fatal("Context passed to RunProcess() has no deadline set.")
 	}
@@ -419,7 +419,6 @@ func RunProcess(t *testing.T,
 		process.WithContext(ctx),
 		process.WithArgs(args),
 		process.WithCmdOptions(attachOutErr(stdOut, stdErr)))
-
 	if err != nil {
 		return fmt.Errorf("failed to spawn %q: %w", processPath, err)
 	}
@@ -549,7 +548,6 @@ func (f *Fixture) executeWithClient(ctx context.Context, command string, disable
 		process.WithContext(ctx),
 		process.WithArgs(args),
 		process.WithCmdOptions(attachOutErr(stdOut, stdErr)))
-
 	if err != nil {
 		return fmt.Errorf("failed to spawn %s: %w", f.binaryName, err)
 	}

pkg/testing/fixture_install.go

@@ -26,12 +26,13 @@ import (
 	"github.com/elastic/elastic-agent-libs/mapstr"
 	agentsystemprocess "github.com/elastic/elastic-agent-system-metrics/metric/system/process"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
+	"github.com/elastic/elastic-agent/pkg/control"
 	"github.com/elastic/elastic-agent/pkg/control/v2/client"
 	"github.com/elastic/elastic-agent/pkg/core/process"
 )
 
 // ErrNotInstalled is returned in cases where Agent isn't installed
-var ErrNotInstalled = errors.New("Elastic Agent is not installed") //nolint:stylecheck // Elastic Agent is a proper noun
+var ErrNotInstalled = errors.New("Elastic Agent is not installed") //nolint:staticcheck // Elastic Agent is a proper noun
 
 // CmdOpts creates vectors of command arguments for different agent commands
 type CmdOpts interface {
@@ -330,12 +331,12 @@ func (f *Fixture) installNoPkgManager(ctx context.Context, installOpts *InstallO
 
 		// environment variable AGENT_KEEP_INSTALLED=true will skip the uninstallation
 		// useful to debug the issue with the Elastic Agent
-		if f.t.Failed() && keepInstalledFlag() {
+		if f.t.Failed() && KeepInstalledFlag() {
 			f.t.Logf("skipping uninstall; test failed and AGENT_KEEP_INSTALLED=true")
 			return
 		}
 
-		if keepInstalledFlag() {
+		if KeepInstalledFlag() {
 			f.t.Logf("ignoring AGENT_KEEP_INSTALLED=true as test succeeded, " +
 				"keeping the agent installed will jeopardise other tests")
 		}
@@ -445,6 +446,19 @@ func getProcesses(t *gotesting.T, regex string) []runningProcess {
 	return processes
 }
 
+func (f *Fixture) SetDebRpmClient() error {
+	workDir := "/var/lib/elastic-agent"
+	socketPath, err := control.AddressFromPath(f.operatingSystem, workDir)
+	if err != nil {
+		return fmt.Errorf("failed to get control protcol address: %w", err)
+	}
+
+	c := client.New(client.WithAddress(socketPath))
+	f.setClient(c)
+
+	return nil
+}
+
 // installDeb installs the prepared Elastic Agent binary from the deb
 // package and registers a t.Cleanup function to uninstall the agent if
 // it hasn't been uninstalled. It also takes care of collecting a
@@ -482,7 +496,7 @@ func (f *Fixture) installDeb(ctx context.Context, installOpts *InstallOpts, shou
 			f.t.Logf("error systemctl stop elastic-agent: %s, output: %s", err, string(out))
 		}
 
-		if keepInstalledFlag() {
+		if KeepInstalledFlag() {
 			f.t.Logf("skipping uninstall; test failed and AGENT_KEEP_INSTALLED=true")
 			return
 		}
@@ -502,6 +516,11 @@ func (f *Fixture) installDeb(ctx context.Context, installOpts *InstallOpts, shou
 		return out, fmt.Errorf("systemctl start elastic-agent failed: %w", err)
 	}
 
+	err = f.SetDebRpmClient()
+	if err != nil {
+		return nil, err
+	}
+
 	if !shouldEnroll {
 		return nil, nil
 	}
@@ -520,11 +539,11 @@ func (f *Fixture) installDeb(ctx context.Context, installOpts *InstallOpts, shou
 	if installOpts.DelayEnroll {
 		enrollArgs = append(enrollArgs, "--delay-enroll")
 	}
-	if installOpts.EnrollOpts.URL != "" {
-		enrollArgs = append(enrollArgs, "--url", installOpts.EnrollOpts.URL)
+	if installOpts.URL != "" {
+		enrollArgs = append(enrollArgs, "--url", installOpts.URL)
 	}
-	if installOpts.EnrollOpts.EnrollmentToken != "" {
-		enrollArgs = append(enrollArgs, "--enrollment-token", installOpts.EnrollOpts.EnrollmentToken)
+	if installOpts.EnrollmentToken != "" {
+		enrollArgs = append(enrollArgs, "--enrollment-token", installOpts.EnrollmentToken)
 	}
 	out, err = exec.CommandContext(ctx, "sudo", enrollArgs...).CombinedOutput()
 	if err != nil {
@@ -593,6 +612,11 @@ func (f *Fixture) installRpm(ctx context.Context, installOpts *InstallOpts, shou
 		return out, fmt.Errorf("systemctl start elastic-agent failed: %w", err)
 	}
 
+	err = f.SetDebRpmClient()
+	if err != nil {
+		return nil, err
+	}
+
 	if !shouldEnroll {
 		return nil, nil
 	}
@@ -611,11 +635,11 @@ func (f *Fixture) installRpm(ctx context.Context, installOpts *InstallOpts, shou
 	if installOpts.DelayEnroll {
 		enrollArgs = append(enrollArgs, "--delay-enroll")
 	}
-	if installOpts.EnrollOpts.URL != "" {
-		enrollArgs = append(enrollArgs, "--url", installOpts.EnrollOpts.URL)
+	if installOpts.URL != "" {
+		enrollArgs = append(enrollArgs, "--url", installOpts.URL)
 	}
-	if installOpts.EnrollOpts.EnrollmentToken != "" {
-		enrollArgs = append(enrollArgs, "--enrollment-token", installOpts.EnrollOpts.EnrollmentToken)
+	if installOpts.EnrollmentToken != "" {
+		enrollArgs = append(enrollArgs, "--enrollment-token", installOpts.EnrollmentToken)
 	}
 	// run sudo elastic-agent enroll
 	out, err = exec.CommandContext(ctx, "sudo", enrollArgs...).CombinedOutput()
@@ -722,7 +746,7 @@ func (f *Fixture) uninstallNoPkgManager(ctx context.Context, uninstallOpts *Unin
 	}
 
 	if err != nil && topPathStats != nil {
-		return out, fmt.Errorf("Elastic Agent is still installed at [%s]", topPath) //nolint:stylecheck // Elastic Agent is a proper noun
+		return out, fmt.Errorf("Elastic Agent is still installed at [%s]", topPath) //nolint:staticcheck // Elastic Agent is a proper noun
 	}
 
 	return out, nil
@@ -825,7 +849,7 @@ func collectDiagFlag() bool {
 	return v
 }
 
-func keepInstalledFlag() bool {
+func KeepInstalledFlag() bool {
 	// failure reports false (ignore error)
 	v, _ := strconv.ParseBool(os.Getenv("AGENT_KEEP_INSTALLED"))
 	return v

pkg/testing/tools/tools.go

@@ -36,6 +36,22 @@ func IsPolicyRevision(ctx context.Context, t *testing.T, client *kibana.Client,
 	}
 }
 
+func GetUninstallToken(ctx context.Context, kibClient *kibana.Client, policyId string) (string, error) {
+	resp, err := kibClient.GetPolicyUninstallTokens(ctx, policyId)
+	if err != nil {
+		return "", fmt.Errorf("failed to fetch uninstall tokens: %w", err)
+	}
+	if len(resp.Items) == 0 {
+		return "", fmt.Errorf("expected non-zero number of tokens")
+	}
+
+	if len(resp.Items[0].Token) == 0 {
+		return "", fmt.Errorf("expected non-empty token")
+	}
+
+	return resp.Items[0].Token, nil
+}
+
 // InstallAgentWithPolicy creates the given policy, enrolls the given agent
 // fixture in Fleet using the default Fleet Server, waits for the agent to be
 // online, and returns the created policy.
@@ -137,7 +153,6 @@ func InstallAgentForPolicyWithToken(ctx context.Context, t *testing.T,
 
 		installOpts.URL = fleetServerURL
 	}
-
 	output, err := agentFixture.Install(ctx, &installOpts)
 	if err != nil {
 		t.Log(string(output))