[7.17](backport #2246) Enable verification with alternative PGP keys (#…

…34604)

[7.17](backport #2246) Enable verification with alternative PGP keys (#34604)

x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl

@@ -99,9 +99,6 @@ inputs:
 #   target_directory: "${path.data}/downloads"
 #   # timeout for downloading package
 #   timeout: 120s
-#   # file path to a public key used for verifying downloaded artifacts
-#   # if not file is present agent will try to load public key from elastic.co website.
-#   pgpfile: "${path.data}/elastic.pgp"
 #   # install_path describes the location of installed packages/programs. It is also used
 #   # for reading program specifications.
 #   install_path: "${path.data}/install"

x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl

@@ -68,9 +68,6 @@ inputs:
 #   target_directory: "${path.data}/downloads"
 #   # timeout for downloading package
 #   timeout: 120s
-#   # file path to a public key used for verifying downloaded artifacts
-#   # if not file is present agent will try to load public key from elastic.co website.
-#   pgpfile: "${path.data}/elastic.pgp"
 #   # install_path describes the location of installed packages/programs. It is also used
 #   # for reading program specifications.
 #   install_path: "${path.data}/install"

x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl

@@ -67,9 +67,6 @@ inputs:
 #   target_directory: "${path.data}/downloads"
 #   # timeout for downloading package
 #   timeout: 120s
-#   # file path to a public key used for verifying downloaded artifacts
-#   # if not file is present agent will try to load public key from elastic.co website.
-#   pgpfile: "${path.data}/elastic.pgp"
 #   # install_path describes the location of installed packages/programs. It is also used
 #   # for reading program specifications.
 #   install_path: "${path.data}/install"

x-pack/elastic-agent/_meta/elastic-agent.fleet.yml

@@ -13,9 +13,6 @@ fleet:
 #   target_directory: "${path.data}/downloads"
 #   # timeout for downloading package
 #   timeout: 120s
-#   # file path to a public key used for verifying downloaded artifacts
-#   # if not file is present Elastic Agent will try to load public key from elastic.co website.
-#   pgpfile: "${path.data}/elastic.pgp"
 #   # install_path describes the location of installed packages/programs. It is also used
 #   # for reading program specifications.
 #   install_path: "${path.data}/install"

x-pack/elastic-agent/_meta/elastic-agent.yml

@@ -64,9 +64,6 @@ inputs:
 #   target_directory: "${path.data}/downloads"
 #   # timeout for downloading package
 #   timeout: 120s
-#   # file path to a public key used for verifying downloaded artifacts
-#   # if not file is present agent will try to load public key from elastic.co website.
-#   pgpfile: "${path.data}/elastic.pgp"
 #   # install_path describes the location of installed packages/programs. It is also used
 #   # for reading program specifications.
 #   install_path: "${path.data}/install"

x-pack/elastic-agent/control.proto

@@ -66,6 +66,16 @@ message UpgradeRequest {
   // If provided the upgrade process will use the provided sourceURI instead of the configured
   // sourceURI in the configuration.
   string sourceURI = 2;
+
+  // (Optional) Overrides predefined behavior for agent package verification.
+  //
+  // If provided Elastic Agent package is not checked for signature during upgrade.
+  bool skipVerify = 3;
+
+  // (Optional) Overrides predefined behavior for agent package verification.
+  //
+  // If provided Elastic Agent package is checked against these pgp keys as well.
+  repeated string pgpBytes = 4;
 }
 
 // A upgrade response message.

x-pack/elastic-agent/elastic-agent.docker.yml

@@ -67,9 +67,6 @@ inputs:
 #   target_directory: "${path.data}/downloads"
 #   # timeout for downloading package
 #   timeout: 120s
-#   # file path to a public key used for verifying downloaded artifacts
-#   # if not file is present agent will try to load public key from elastic.co website.
-#   pgpfile: "${path.data}/elastic.pgp"
 #   # install_path describes the location of installed packages/programs. It is also used
 #   # for reading program specifications.
 #   install_path: "${path.data}/install"

x-pack/elastic-agent/elastic-agent.reference.yml

@@ -74,9 +74,6 @@ inputs:
 #   target_directory: "${path.data}/downloads"
 #   # timeout for downloading package
 #   timeout: 120s
-#   # file path to a public key used for verifying downloaded artifacts
-#   # if not file is present agent will try to load public key from elastic.co website.
-#   pgpfile: "${path.data}/elastic.pgp"
 #   # install_path describes the location of installed packages/programs. It is also used
 #   # for reading program specifications.
 #   install_path: "${path.data}/install"

x-pack/elastic-agent/elastic-agent.yml

@@ -105,9 +105,6 @@ inputs:
 #   target_directory: "${path.data}/downloads"
 #   # timeout for downloading package
 #   timeout: 120s
-#   # file path to a public key used for verifying downloaded artifacts
-#   # if not file is present agent will try to load public key from elastic.co website.
-#   pgpfile: "${path.data}/elastic.pgp"
 #   # install_path describes the location of installed packages/programs. It is also used
 #   # for reading program specifications.
 #   install_path: "${path.data}/install"

x-pack/elastic-agent/magefile.go

@@ -407,7 +407,16 @@ func Config() {
 
 // ControlProto generates pkg/agent/control/proto module.
 func ControlProto() error {
-	return sh.RunV("protoc", "--go_out=plugins=grpc:.", "control.proto")
+	err := sh.RunV("protoc", "--go_out=plugins=grpc:.", "control.proto")
+	if err == nil {
+		return nil
+	}
+
+	return sh.RunV(
+		"protoc",
+		"--go_out=pkg/agent/control/proto/", "--go_opt=paths=source_relative",
+		"--go-grpc_out=pkg/agent/control/proto/", "--go-grpc_opt=paths=source_relative",
+		"control.proto")
 }
 
 // BuildSpec make sure that all the suppported program spec are built into the binary.

x-pack/elastic-agent/pkg/agent/application/pipeline/actions/handlers/handler_action_upgrade.go

@@ -38,7 +38,7 @@ func (h *Upgrade) Handle(ctx context.Context, a fleetapi.Action, acker store.Fle
 		return fmt.Errorf("invalid type, expected ActionUpgrade and received %T", a)
 	}
 
-	_, err := h.upgrader.Upgrade(ctx, &upgradeAction{action}, true)
+	_, err := h.upgrader.Upgrade(ctx, &upgradeAction{action}, true, false)
 	return err
 }
 

x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go

@@ -20,7 +20,7 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
 )
 
-func (u *Upgrader) downloadArtifact(ctx context.Context, version, sourceURI string) (string, error) {
+func (u *Upgrader) downloadArtifact(ctx context.Context, version, sourceURI string, skipVerifyOverride bool, pgpBytes ...string) (string, error) {
 	// do not update source config
 	settings := *u.settings
 	if sourceURI != "" {
@@ -33,11 +33,6 @@ func (u *Upgrader) downloadArtifact(ctx context.Context, version, sourceURI stri
 		}
 	}
 
-	verifier, err := newVerifier(version, u.log, &settings)
-	if err != nil {
-		return "", errors.New(err, "initiating verifier")
-	}
-
 	fetcher, err := newDownloader(version, u.log, &settings)
 	if err != nil {
 		return "", errors.New(err, "initiating fetcher")
@@ -48,7 +43,16 @@ func (u *Upgrader) downloadArtifact(ctx context.Context, version, sourceURI stri
 		return "", errors.New(err, "failed upgrade of agent binary")
 	}
 
-	matches, err := verifier.Verify(agentSpec, version, true)
+	if skipVerifyOverride {
+		return path, nil
+	}
+
+	verifier, err := newVerifier(version, u.log, &settings)
+	if err != nil {
+		return "", errors.New(err, "initiating verifier")
+	}
+
+	matches, err := verifier.Verify(agentSpec, version, true, pgpBytes...)
 	if err != nil {
 		return "", errors.New(err, "failed verification of agent binary")
 	}

x-pack/elastic-agent/pkg/agent/application/upgrade/step_mark.go

@@ -39,7 +39,7 @@ type UpdateMarker struct {
 }
 
 // markUpgrade marks update happened so we can handle grace period
-func (h *Upgrader) markUpgrade(ctx context.Context, hash string, action Action) error {
+func (u *Upgrader) markUpgrade(ctx context.Context, hash string, action Action) error {
 	prevVersion := release.Version()
 	prevHash := release.Commit()
 	if len(prevHash) > hashLen {

x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go

@@ -106,7 +106,7 @@ func (u *Upgrader) Upgradeable() bool {
 
 // Upgrade upgrades running agent, function returns shutdown callback if some needs to be executed for cases when
 // reexec is called by caller.
-func (u *Upgrader) Upgrade(ctx context.Context, a Action, reexecNow bool) (_ reexec.ShutdownCallbackFn, err error) {
+func (u *Upgrader) Upgrade(ctx context.Context, a Action, reexecNow bool, skipVerifyOverride bool, pgpBytes ...string) (_ reexec.ShutdownCallbackFn, err error) {
 	// report failed
 	defer func() {
 		if err != nil {
@@ -123,15 +123,15 @@ func (u *Upgrader) Upgrade(ctx context.Context, a Action, reexecNow bool) (_ ree
 	}
 
 	if u.caps != nil {
-		if _, err := u.caps.Apply(a); err == capabilities.ErrBlocked {
+		if _, err := u.caps.Apply(a); errors.Is(err, capabilities.ErrBlocked) {
 			return nil, nil
 		}
 	}
 
 	u.reportUpdating(a.Version())
 
 	sourceURI, err := u.sourceURI(a.Version(), a.SourceURI())
-	archivePath, err := u.downloadArtifact(ctx, a.Version(), sourceURI)
+	archivePath, err := u.downloadArtifact(ctx, a.Version(), sourceURI, skipVerifyOverride, pgpBytes...)
 	if err != nil {
 		return nil, err
 	}
@@ -148,7 +148,7 @@ func (u *Upgrader) Upgrade(ctx context.Context, a Action, reexecNow bool) (_ ree
 	if strings.HasPrefix(release.Commit(), newHash) {
 		// not an error
 		if action := a.FleetAction(); action != nil {
-			u.ackAction(ctx, action)
+			_ = u.ackAction(ctx, action)
 		}
 		u.log.Warn("upgrading to same version")
 		return nil, nil
@@ -236,7 +236,7 @@ func (u *Upgrader) ackAction(ctx context.Context, action fleetapi.Action) error
 // and state is changed to FAILED
 func (u *Upgrader) reportFailure(ctx context.Context, action fleetapi.Action, err error) {
 	// ack action
-	u.acker.Ack(ctx, action)
+	_ = u.acker.Ack(ctx, action)
 
 	// report failure
 	u.reporter.OnStateChange(
@@ -257,8 +257,8 @@ func (u *Upgrader) reportUpdating(version string) {
 }
 
 func rollbackInstall(ctx context.Context, hash string) {
-	os.RemoveAll(filepath.Join(paths.Data(), fmt.Sprintf("%s-%s", agentName, hash)))
-	ChangeSymlink(ctx, release.ShortCommit())
+	_ = os.RemoveAll(filepath.Join(paths.Data(), fmt.Sprintf("%s-%s", agentName, hash)))
+	_ = ChangeSymlink(ctx, release.ShortCommit())
 }
 
 func copyActionStore(newHash string) error {

x-pack/elastic-agent/pkg/agent/cmd/upgrade.go

@@ -7,16 +7,26 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
 
 	"github.com/spf13/cobra"
 
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/client"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli"
 )
 
+const (
+	flagSourceURI    = "source-uri"
+	flagSkipVerify   = "skip-verify"
+	flagPGPBytes     = "pgp"
+	flagPGPBytesPath = "pgp-path"
+	flagPGPBytesURI  = "pgp-uri"
+)
+
 func newUpgradeCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "upgrade <version>",
@@ -30,7 +40,11 @@ func newUpgradeCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Comman
 		},
 	}
 
-	cmd.Flags().StringP("source-uri", "s", "", "Source URI to download the new version from")
+	cmd.Flags().StringP(flagSourceURI, "s", "", "Source URI to download the new version from")
+	cmd.Flags().BoolP(flagSkipVerify, "", false, "Skips package verification")
+	cmd.Flags().String(flagPGPBytes, "", "PGP to use for package verification")
+	cmd.Flags().String(flagPGPBytesURI, "", "Path to a web location containing PGP to use for package verification")
+	cmd.Flags().String(flagPGPBytesPath, "", "Path to a file containing PGP to use for package verification")
 
 	return cmd
 }
@@ -39,15 +53,46 @@ func upgradeCmd(streams *cli.IOStreams, cmd *cobra.Command, args []string) error
 	fmt.Fprintln(streams.Out, "The upgrade process of Elastic Agent is currently EXPERIMENTAL and should not be used in production")
 
 	version := args[0]
-	sourceURI, _ := cmd.Flags().GetString("source-uri")
+	sourceURI, _ := cmd.Flags().GetString(flagSourceURI)
+
+	skipVerification, _ := cmd.Flags().GetBool(flagSkipVerify)
+	var pgpChecks []string
+	if !skipVerification {
+		// get local PGP
+		pgpPath, _ := cmd.Flags().GetString(flagPGPBytesPath)
+		if len(pgpPath) > 0 {
+			content, err := ioutil.ReadFile(pgpPath)
+			if err != nil {
+				return errors.New(err, "failed to read pgp file")
+			}
+			if len(content) > 0 {
+				pgpChecks = append(pgpChecks, download.PgpSourceRawPrefix+string(content))
+			}
+		}
+
+		pgpBytes, _ := cmd.Flags().GetString(flagPGPBytes)
+		if len(pgpBytes) > 0 {
+			pgpChecks = append(pgpChecks, download.PgpSourceRawPrefix+pgpBytes)
+		}
+
+		pgpURI, _ := cmd.Flags().GetString(flagPGPBytesURI)
+		if len(pgpURI) > 0 {
+			if uriErr := download.CheckValidDownloadURI(pgpURI); uriErr != nil {
+				return uriErr
+			}
+
+			// URI is parsed later with proper TLS and Proxy config within downloader
+			pgpChecks = append(pgpChecks, download.PgpSourceURIPrefix+pgpURI)
+		}
+	}
 
 	c := client.New()
 	err := c.Connect(context.Background())
 	if err != nil {
 		return errors.New(err, "Failed communicating to running daemon", errors.TypeNetwork, errors.M("socket", control.Address()))
 	}
 	defer c.Disconnect()
-	version, err = c.Upgrade(context.Background(), version, sourceURI)
+	version, err = c.Upgrade(context.Background(), version, sourceURI, skipVerification, pgpChecks...)
 	if err != nil {
 		return errors.New(err, "Failed trigger upgrade of daemon")
 	}

x-pack/elastic-agent/pkg/agent/control/client/client.go

@@ -92,7 +92,7 @@ type Client interface {
 	// Restart triggers restarting the current running daemon.
 	Restart(ctx context.Context) error
 	// Upgrade triggers upgrade of the current running daemon.
-	Upgrade(ctx context.Context, version string, sourceURI string) (string, error)
+	Upgrade(ctx context.Context, version string, sourceURI string, skipVerify bool, pgpBytes ...string) (string, error)
 	// ProcMeta gathers running process meta-data.
 	ProcMeta(ctx context.Context) ([]ProcMeta, error)
 }
@@ -192,10 +192,12 @@ func (c *client) Restart(ctx context.Context) error {
 }
 
 // Upgrade triggers upgrade of the current running daemon.
-func (c *client) Upgrade(ctx context.Context, version string, sourceURI string) (string, error) {
+func (c *client) Upgrade(ctx context.Context, version string, sourceURI string, skipVerify bool, pgpBytes ...string) (string, error) {
 	res, err := c.client.Upgrade(ctx, &proto.UpgradeRequest{
-		Version:   version,
-		SourceURI: sourceURI,
+		Version:    version,
+		SourceURI:  sourceURI,
+		SkipVerify: skipVerify,
+		PgpBytes:   pgpBytes,
 	})
 	if err != nil {
 		return "", err