[Ingest Manager] Agent includes pgp file

x-pack/elastic-agent/magefile.go

@@ -10,10 +10,14 @@ import (
  "context"
  "errors"
  "fmt"
+ "io/ioutil"
+ "net/http"
+ "net/url"
  "os"
  "os/exec"
  "path/filepath"
  "runtime"
+ "strconv"
  "strings"
  "time"
 
@@ -40,6 +44,7 @@ const (
  buildDir = "build"
  metaDir = "_meta"
  snapshotEnv = "SNAPSHOT"
+ devEnv = "DEV"
  configFile = "elastic-agent.yml"
  agentDropPath = "AGENT_DROP_PATH"
 )
@@ -620,6 +625,23 @@ func buildVars() map[string]string {
  isSnapshot, _ := os.LookupEnv(snapshotEnv)
  vars["github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release.snapshot"] = isSnapshot
 
+ fetchPgp := true
+ if isDevFlag, devFound := os.LookupEnv(devEnv); devFound {
+ if isDev, err := strconv.ParseBool(isDevFlag); err == nil && isDev {
+ vars["github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release.allowEmptyPgp"] = "true"
+ fetchPgp = false
+ }
+ }
+ fmt.Println("fetching pgp", fetchPgp)
+
+ if fetchPgp {
+ pgp, err := loadPGPFromWeb()
+ if err != nil {
+ panic(err)
+ }
+ vars["github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release.escPgp"] = string(pgp)
+ }
+
  return vars
 }
 
@@ -628,3 +650,23 @@ func injectBuildVars(m map[string]string) {
  m[k] = v
  }
 }
+
+func loadPGPFromWeb() (string, error) {
+ const publicKeyURI = "https://artifacts.elastic.co/GPG-KEY-elasticsearch"
+
+ resp, err := http.Get(publicKeyURI)
+ if err != nil {
+ return "", fmt.Errorf("failed loading public key: %v", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != 200 {
+ return "", fmt.Errorf("call to '%s' returned unsuccessful status code: %d", publicKeyURI, resp.StatusCode)
+ }
+
+ rawPgp, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return "", err
+ }
+ return url.PathEscape(string(rawPgp)), nil
+}

x-pack/elastic-agent/pkg/agent/application/stream.go

@@ -19,6 +19,7 @@ import (
  "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/monitoring"
  "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/server"
  "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/state"
+ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
 )
 
 type operatorStream struct {
@@ -56,7 +57,8 @@ func streamFactory(ctx context.Context, cfg *configuration.SettingsConfig, srv *
 
 func newOperator(ctx context.Context, log *logger.Logger, id routingKey, config *configuration.SettingsConfig, srv *server.Server, r state.Reporter, m monitoring.Monitor) (*operation.Operator, error) {
  fetcher := downloader.NewDownloader(log, config.DownloadConfig)
- verifier, err := downloader.NewVerifier(log, config.DownloadConfig)
+ allowEmptyPgp, pgp := release.PGP()
+ verifier, err := downloader.NewVerifier(log, config.DownloadConfig, allowEmptyPgp, pgp)
  if err != nil {
  return nil, errors.New(err, "initiating verifier")
  }

x-pack/elastic-agent/pkg/artifact/config.go

@@ -30,10 +30,6 @@ type Config struct {
  // Timeout: timeout for downloading package
  Timeout time.Duration `json:"timeout" config:"timeout"`
 
- // PgpFile: filepath to a public key used for verifying downloaded artifacts
- // if not file is present elastic-agent will try to load public key from elastic.co website.
- PgpFile string `json:"pgpfile" config:"pgpfile"`
-
  // InstallPath: path to the directory containing installed packages
  InstallPath string `yaml:"installPath" config:"install_path"`
 
@@ -52,7 +48,6 @@ func DefaultConfig() *Config {
  SourceURI: "https://artifacts.elastic.co/downloads/",
  TargetDirectory: filepath.Join(dataPath, "downloads"),
  Timeout: 30 * time.Second,
- PgpFile: filepath.Join(dataPath, "elastic.pgp"),
  InstallPath: filepath.Join(dataPath, "install"),
  }
 }

x-pack/elastic-agent/pkg/artifact/download/fs/verifier.go

@@ -14,7 +14,6 @@ import (
  "os"
  "path/filepath"
  "strings"
- "sync"
 
  "golang.org/x/crypto/openpgp"
 
@@ -29,15 +28,22 @@ const (
 // Verifier verifies a downloaded package by comparing with public ASC
 // file from elastic.co website.
 type Verifier struct {
- config *artifact.Config
- pgpBytes []byte
+ config *artifact.Config
+ pgpBytes []byte
+ allowEmptyPgp bool
 }
 
 // NewVerifier create a verifier checking downloaded package on preconfigured
 // location agains a key stored on elastic.co website.
-func NewVerifier(config *artifact.Config) (*Verifier, error) {
+func NewVerifier(config *artifact.Config, allowEmptyPgp bool, pgp []byte) (*Verifier, error) {
+ if len(pgp) == 0 && !allowEmptyPgp {
+ return nil, errors.New("expecting PGP but retrieved none", errors.TypeSecurity)
+ }
+
  v := &Verifier{
- config: config,
+ config: config,
+ allowEmptyPgp: allowEmptyPgp,
+ pgpBytes: pgp,
  }
 
  return v, nil
@@ -58,9 +64,10 @@ func (v *Verifier) Verify(programName, version string) (bool, error) {
  // remove bits so they can be redownloaded
  os.Remove(fullPath)
  os.Remove(fullPath + ".sha512")
+ return isMatch, err
  }
 
- return isMatch, err
+ return v.verifyAsc(filename, fullPath)
 }
 
 func (v *Verifier) verifyHash(filename, fullPath string) (bool, error) {
@@ -106,15 +113,9 @@ func (v *Verifier) verifyHash(filename, fullPath string) (bool, error) {
 }
 
 func (v *Verifier) verifyAsc(filename, fullPath string) (bool, error) {
- var err error
- var pgpBytesLoader sync.Once
-
- pgpBytesLoader.Do(func() {
- err = v.loadPGP(v.config.PgpFile)
- })
-
- if err != nil {
- return false, errors.New(err, "loading PGP")
+ if len(v.pgpBytes) == 0 {
+ // no pgp available skip verification process
+ return true, nil
  }
 
  ascBytes, err := v.getPublicAsc(filename)
@@ -153,18 +154,3 @@ func (v *Verifier) getPublicAsc(filename string) ([]byte, error) {
 
  return b, nil
 }
-
-func (v *Verifier) loadPGP(file string) error {
- var err error
-
- if file == "" {
- return errors.New("pgp file not specified for verifier", errors.TypeConfig)
- }
-
- v.pgpBytes, err = ioutil.ReadFile(file)
- if err != nil {
- return errors.New(err, errors.TypeFilesystem)
- }
-
- return nil
-}

x-pack/elastic-agent/pkg/artifact/download/fs/verifier_test.go

@@ -59,7 +59,7 @@ func TestFetchVerify(t *testing.T) {
  assert.NoError(t, err)
 
  downloader := NewDownloader(config)
- verifier, err := NewVerifier(config)
+ verifier, err := NewVerifier(config, true, nil)
  assert.NoError(t, err)
 
  // first download verify should fail:
@@ -157,7 +157,7 @@ func TestVerify(t *testing.T) {
  t.Fatal(err)
  }
 
- testVerifier, err := NewVerifier(config)
+ testVerifier, err := NewVerifier(config, true, nil)
  if err != nil {
  t.Fatal(err)
  }

x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go

@@ -105,7 +105,7 @@ func TestVerify(t *testing.T) {
  t.Fatal(err)
  }
 
- testVerifier, err := NewVerifier(config)
+ testVerifier, err := NewVerifier(config, true, nil)
  if err != nil {
  t.Fatal(err)
  }

x-pack/elastic-agent/pkg/artifact/download/http/verifier.go

@@ -16,7 +16,6 @@ import (
  "os"
  "path"
  "strings"
- "sync"
 
  "golang.org/x/crypto/openpgp"
 
@@ -32,20 +31,27 @@ const (
 // Verifier verifies a downloaded package by comparing with public ASC
 // file from elastic.co website.
 type Verifier struct {
- config *artifact.Config
- client http.Client
- pgpBytes []byte
+ config *artifact.Config
+ client http.Client
+ pgpBytes []byte
+ allowEmptyPgp bool
 }
 
 // NewVerifier create a verifier checking downloaded package on preconfigured
 // location agains a key stored on elastic.co website.
-func NewVerifier(config *artifact.Config) (*Verifier, error) {
+func NewVerifier(config *artifact.Config, allowEmptyPgp bool, pgp []byte) (*Verifier, error) {
+ if len(pgp) == 0 && !allowEmptyPgp {
+ return nil, errors.New("expecting PGP but retrieved none", errors.TypeSecurity)
+ }
+
  client := http.Client{Timeout: config.Timeout}
  rtt := withHeaders(client.Transport, headers)
  client.Transport = rtt
  v := &Verifier{
- config: config,
- client: client,
+ config: config,
+ client: client,
+ allowEmptyPgp: allowEmptyPgp,
+ pgpBytes: pgp,
  }
 
  return v, nil
@@ -71,9 +77,10 @@ func (v *Verifier) Verify(programName, version string) (bool, error) {
  // remove bits so they can be redownloaded
  os.Remove(fullPath)
  os.Remove(fullPath + ".sha512")
+ return isMatch, err
  }
 
- return isMatch, err
+ return v.verifyAsc(programName, version)
 }
 
 func (v *Verifier) verifyHash(filename, fullPath string) (bool, error) {
@@ -120,15 +127,9 @@ func (v *Verifier) verifyHash(filename, fullPath string) (bool, error) {
 }
 
 func (v *Verifier) verifyAsc(programName, version string) (bool, error) {
- var err error
- var pgpBytesLoader sync.Once
-
- pgpBytesLoader.Do(func() {
- err = v.loadPGP(v.config.PgpFile)
- })
-
- if err != nil {
- return false, errors.New(err, "loading PGP")
+ if len(v.pgpBytes) == 0 {
+ // no pgp available skip verification process
+ return true, nil
  }
 
  filename, err := artifact.GetArtifactName(programName, version, v.config.OS(), v.config.Arch())
@@ -202,33 +203,3 @@ func (v *Verifier) getPublicAsc(sourceURI string) ([]byte, error) {
 
  return ioutil.ReadAll(resp.Body)
 }
-
-func (v *Verifier) loadPGP(file string) error {
- var err error
-
- if file == "" {
- v.pgpBytes, err = v.loadPGPFromWeb()
- return err
- }
-
- v.pgpBytes, err = ioutil.ReadFile(file)
- if err != nil {
- return errors.New(err, errors.TypeFilesystem, errors.M(errors.MetaKeyPath, file))
- }
-
- return nil
-}
-
-func (v *Verifier) loadPGPFromWeb() ([]byte, error) {
- resp, err := v.client.Get(publicKeyURI)
- if err != nil {
- return nil, errors.New(err, "failed loading public key", errors.TypeNetwork, errors.M(errors.MetaKeyURI, publicKeyURI))
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != 200 {
- return nil, errors.New(fmt.Sprintf("call to '%s' returned unsuccessful status code: %d", publicKeyURI, resp.StatusCode), errors.TypeNetwork, errors.M(errors.MetaKeyURI, publicKeyURI))
- }
-
- return ioutil.ReadAll(resp.Body)
-}

x-pack/elastic-agent/pkg/artifact/download/localremote/verifier.go

@@ -17,26 +17,26 @@ import (
 
 // NewVerifier creates a downloader which first checks local directory
 // and then fallbacks to remote if configured.
-func NewVerifier(log *logger.Logger, config *artifact.Config) (download.Verifier, error) {
+func NewVerifier(log *logger.Logger, config *artifact.Config, allowEmptyPgp bool, pgp []byte) (download.Verifier, error) {
  verifiers := make([]download.Verifier, 0, 3)
 
- fsVer, err := fs.NewVerifier(config)
+ fsVer, err := fs.NewVerifier(config, allowEmptyPgp, pgp)
  if err != nil {
  return nil, err
  }
  verifiers = append(verifiers, fsVer)
 
  // try snapshot repo before official
  if release.Snapshot() {
- snapshotVerifier, err := snapshot.NewVerifier(config)
+ snapshotVerifier, err := snapshot.NewVerifier(config, allowEmptyPgp, pgp)
  if err != nil {
  log.Error(err)
  } else {
  verifiers = append(verifiers, snapshotVerifier)
  }
  }
 
- remoteVer, err := http.NewVerifier(config)
+ remoteVer, err := http.NewVerifier(config, allowEmptyPgp, pgp)
  if err != nil {
  return nil, err
  }

x-pack/elastic-agent/pkg/artifact/download/snapshot/downloader.go

@@ -38,7 +38,6 @@ func snapshotConfig(config *artifact.Config) (*artifact.Config, error) {
  SourceURI: snapshotURI,
  TargetDirectory: config.TargetDirectory,
  Timeout: config.Timeout,
- PgpFile: config.PgpFile,
  InstallPath: config.InstallPath,
  DropPath: config.DropPath,
  }, nil

x-pack/elastic-agent/pkg/artifact/download/snapshot/verifier.go

@@ -12,10 +12,10 @@ import (
 
 // NewVerifier creates a downloader which first checks local directory
 // and then fallbacks to remote if configured.
-func NewVerifier(config *artifact.Config, downloaders ...download.Downloader) (download.Verifier, error) {
+func NewVerifier(config *artifact.Config, allowEmptyPgp bool, pgp []byte) (download.Verifier, error) {
  cfg, err := snapshotConfig(config)
  if err != nil {
  return nil, err
  }
- return http.NewVerifier(cfg)
+ return http.NewVerifier(cfg, allowEmptyPgp, pgp)
 }