From 8c992c508b1bb3c690034b675110e2d595afd19a Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 29 Sep 2020 07:11:22 -0400 Subject: [PATCH 01/17] Add IP validation to Security module (#21325) For event 4778 (A session was reconnected to a Window Station) the `winlog.event_data.ClientAddress` could be "LOCAL" which is obviosuly not a valid IP so we don't want to copy it into `source.ip` in that case. Fixes #19627 --- CHANGELOG.next.asciidoc | 1 + .../winlogbeat/module/security/config/winlogbeat-security.js | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 58ca2acb35c..76a7f1edeb6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -367,6 +367,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix invalid IP addresses in DNS query results from Sysmon data. {issue}18432[18432] {pull}18436[18436] - Fields from Winlogbeat modules were not being included in index templates and patterns. {pull}18983[18983] +- Add source.ip validation for event ID 4778 in the Security module. {issue}19627[19627] *Functionbeat* diff --git a/x-pack/winlogbeat/module/security/config/winlogbeat-security.js b/x-pack/winlogbeat/module/security/config/winlogbeat-security.js index 9a117a42f6f..9a0899165b9 100644 --- a/x-pack/winlogbeat/module/security/config/winlogbeat-security.js +++ b/x-pack/winlogbeat/module/security/config/winlogbeat-security.js @@ -1479,11 +1479,12 @@ var security = (function () { fields: [ {from: "winlog.event_data.AccountName", to: "user.name"}, {from: "winlog.event_data.AccountDomain", to: "user.domain"}, - {from: "winlog.event_data.ClientAddress", to: "source.ip"}, + {from: "winlog.event_data.ClientAddress", to: "source.ip", type: "ip"}, {from: "winlog.event_data.ClientName", to: "source.domain"}, {from: "winlog.event_data.LogonID", to: "winlog.logon.id"}, ], ignore_missing: true, + fail_on_error: false, }) .Add(function(evt) { var user = evt.Get("winlog.event_data.AccountName"); From 0cbf4e50a5e12f331529007dd88c77b6f65b9771 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Tue, 29 Sep 2020 13:13:14 +0200 Subject: [PATCH 02/17] feat: prepare release pipelines (#21238) * feat: prepare release pipelines * Apply suggestions from code review --- .ci/jobs/beats-release-minor-major.yml | 20 ++++++++++++++++++++ .ci/jobs/beats-release-patch.yml | 20 ++++++++++++++++++++ .ci/jobs/folders.yml | 5 +++++ 3 files changed, 45 insertions(+) create mode 100644 .ci/jobs/beats-release-minor-major.yml create mode 100644 .ci/jobs/beats-release-patch.yml diff --git a/.ci/jobs/beats-release-minor-major.yml b/.ci/jobs/beats-release-minor-major.yml new file mode 100644 index 00000000000..91c7d105fb8 --- /dev/null +++ b/.ci/jobs/beats-release-minor-major.yml @@ -0,0 +1,20 @@ +--- +- job: + name: Beats/Release/beats-release-minor-major + display-name: 'Prepare Major/minor Release' + description: 'Automate the steps to prepare a new Release branch' + view: Beats + project-type: pipeline + pipeline-scm: + script-path: release_scripts/pipeline-release-minor-major.groovy + scm: + - git: + url: git@github.com:elastic/ingest-dev.git + refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/* + wipe-workspace: 'True' + name: origin + shallow-clone: true + credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba + reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git + branches: + - master diff --git a/.ci/jobs/beats-release-patch.yml b/.ci/jobs/beats-release-patch.yml new file mode 100644 index 00000000000..4d205f79647 --- /dev/null +++ b/.ci/jobs/beats-release-patch.yml @@ -0,0 +1,20 @@ +--- +- job: + name: Beats/Release/beats-release-patch + display-name: 'Prepare Patch Release' + description: 'Automate the steps to prepare a new Patch' + view: Beats + project-type: pipeline + pipeline-scm: + script-path: release_scripts/pipeline-release-patch.groovy + scm: + - git: + url: git@github.com:elastic/ingest-dev.git + refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/* + wipe-workspace: 'True' + name: origin + shallow-clone: true + credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba + reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git + branches: + - master diff --git a/.ci/jobs/folders.yml b/.ci/jobs/folders.yml index 60b6e3eff92..4b6cc5c4944 100644 --- a/.ci/jobs/folders.yml +++ b/.ci/jobs/folders.yml @@ -4,3 +4,8 @@ name: Beats description: Beats project-type: folder + +- job: + name: Beats/Release + description: Jobs for release preparation + project-type: folder From 8f9d54bef44f9745c958bb727aed4c0e180c93cd Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Tue, 29 Sep 2020 16:20:29 +0200 Subject: [PATCH 03/17] [Filebeat][httpjson] Make httpjson use cursor input when using date cursor (#20751) * Fix duplicate import * Move config to its own package * Minor improvements * Fix tests * Create input manager * Change requester to accept and store a cursor * Modify input to be embedded * Create stateless and cursor inputs * Initialize new input manager on publish * Add changelog entry and format files * Move test data folder * Change tests * Apply requested changes --- CHANGELOG.next.asciidoc | 1 + filebeat/input/v2/input-cursor/manager.go | 7 +- .../filebeat/input/default-inputs/inputs.go | 2 +- x-pack/filebeat/input/httpjson/config.go | 58 +++++------ .../filebeat/input/httpjson/config_oauth.go | 73 +++++++------- .../input/httpjson/config_oauth_test.go | 38 ++++---- x-pack/filebeat/input/httpjson/config_test.go | 18 ++-- x-pack/filebeat/input/httpjson/date_cursor.go | 41 ++++---- x-pack/filebeat/input/httpjson/input.go | 96 +++++++++---------- .../filebeat/input/httpjson/input_cursor.go | 67 +++++++++++++ .../filebeat/input/httpjson/input_manager.go | 49 ++++++++++ .../input/httpjson/input_stateless.go | 58 +++++++++++ .../{httpjson_test.go => input_test.go} | 17 ++-- x-pack/filebeat/input/httpjson/pagination.go | 4 +- .../input/httpjson/pagination_test.go | 2 +- .../filebeat/input/httpjson/rate_limiter.go | 2 +- x-pack/filebeat/input/httpjson/requester.go | 36 +++++-- 17 files changed, 385 insertions(+), 184 deletions(-) create mode 100644 x-pack/filebeat/input/httpjson/input_cursor.go create mode 100644 x-pack/filebeat/input/httpjson/input_manager.go create mode 100644 x-pack/filebeat/input/httpjson/input_stateless.go rename x-pack/filebeat/input/httpjson/{httpjson_test.go => input_test.go} (96%) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 76a7f1edeb6..1b5b1bd16bb 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -597,6 +597,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add type and sub_type to panw panos fileset {pull}20912[20912] - Always attempt community_id processor on zeek module {pull}21155[21155] - Add related.hosts ecs field to all modules {pull}21160[21160] +- Keep cursor state between httpjson input restarts {pull}20751[20751] *Heartbeat* diff --git a/filebeat/input/v2/input-cursor/manager.go b/filebeat/input/v2/input-cursor/manager.go index 2a4310dc778..766d6f17fa0 100644 --- a/filebeat/input/v2/input-cursor/manager.go +++ b/filebeat/input/v2/input-cursor/manager.go @@ -26,7 +26,6 @@ import ( "github.com/elastic/go-concert/unison" - input "github.com/elastic/beats/v7/filebeat/input/v2" v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" @@ -145,7 +144,7 @@ func (cim *InputManager) shutdown() { // Create builds a new v2.Input using the provided Configure function. // The Input will run a go-routine per source that has been configured. -func (cim *InputManager) Create(config *common.Config) (input.Input, error) { +func (cim *InputManager) Create(config *common.Config) (v2.Input, error) { if err := cim.init(); err != nil { return nil, err } @@ -180,7 +179,7 @@ func (cim *InputManager) Create(config *common.Config) (input.Input, error) { // Lock locks a key for exclusive access and returns an resource that can be used to modify // the cursor state and unlock the key. -func (cim *InputManager) lock(ctx input.Context, key string) (*resource, error) { +func (cim *InputManager) lock(ctx v2.Context, key string) (*resource, error) { resource := cim.store.Get(key) err := lockResource(ctx.Logger, resource, ctx.Cancelation) if err != nil { @@ -190,7 +189,7 @@ func (cim *InputManager) lock(ctx input.Context, key string) (*resource, error) return resource, nil } -func lockResource(log *logp.Logger, resource *resource, canceler input.Canceler) error { +func lockResource(log *logp.Logger, resource *resource, canceler v2.Canceler) error { if !resource.lock.TryLock() { log.Infof("Resource '%v' currently in use, waiting...", resource.key) err := resource.lock.LockContext(canceler) diff --git a/x-pack/filebeat/input/default-inputs/inputs.go b/x-pack/filebeat/input/default-inputs/inputs.go index 1fe245b80f7..cd8562560da 100644 --- a/x-pack/filebeat/input/default-inputs/inputs.go +++ b/x-pack/filebeat/input/default-inputs/inputs.go @@ -27,7 +27,7 @@ func xpackInputs(info beat.Info, log *logp.Logger, store beater.StateStore) []v2 return []v2.Plugin{ cloudfoundry.Plugin(), http_endpoint.Plugin(), - httpjson.Plugin(), + httpjson.Plugin(log, store), o365audit.Plugin(log, store), } } diff --git a/x-pack/filebeat/input/httpjson/config.go b/x-pack/filebeat/input/httpjson/config.go index 95ca205be0d..ee1445b8a3d 100644 --- a/x-pack/filebeat/input/httpjson/config.go +++ b/x-pack/filebeat/input/httpjson/config.go @@ -17,9 +17,9 @@ import ( "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" ) -// Config contains information about httpjson configuration +// config contains information about httpjson configuration type config struct { - OAuth2 *OAuth2 `config:"oauth2"` + OAuth2 *oauth2Config `config:"oauth2"` APIKey string `config:"api_key"` AuthenticationScheme string `config:"authentication_scheme"` HTTPClientTimeout time.Duration `config:"http_client_timeout"` @@ -30,21 +30,21 @@ type config struct { JSONObjects string `config:"json_objects_array"` SplitEventsBy string `config:"split_events_by"` NoHTTPBody bool `config:"no_http_body"` - Pagination *Pagination `config:"pagination"` - RateLimit *RateLimit `config:"rate_limit"` + Pagination *paginationConfig `config:"pagination"` + RateLimit *rateLimitConfig `config:"rate_limit"` RetryMax int `config:"retry.max_attempts"` RetryWaitMin time.Duration `config:"retry.wait_min"` RetryWaitMax time.Duration `config:"retry.wait_max"` TLS *tlscommon.Config `config:"ssl"` - URL *URL `config:"url" validate:"required"` - DateCursor *DateCursor `config:"date_cursor"` + URL *urlConfig `config:"url" validate:"required"` + DateCursor *dateCursorConfig `config:"date_cursor"` } // Pagination contains information about httpjson pagination settings -type Pagination struct { +type paginationConfig struct { Enabled *bool `config:"enabled"` ExtraBodyContent common.MapStr `config:"extra_body_content"` - Header *Header `config:"header"` + Header *headerConfig `config:"header"` IDField string `config:"id_field"` RequestField string `config:"req_field"` URLField string `config:"url_field"` @@ -52,76 +52,76 @@ type Pagination struct { } // IsEnabled returns true if the `enable` field is set to true in the yaml. -func (p *Pagination) IsEnabled() bool { +func (p *paginationConfig) isEnabled() bool { return p != nil && (p.Enabled == nil || *p.Enabled) } // HTTP Header information for pagination -type Header struct { +type headerConfig struct { FieldName string `config:"field_name" validate:"required"` RegexPattern *regexp.Regexp `config:"regex_pattern" validate:"required"` } // HTTP Header Rate Limit information -type RateLimit struct { +type rateLimitConfig struct { Limit string `config:"limit"` Reset string `config:"reset"` Remaining string `config:"remaining"` } -type DateCursor struct { - Enabled *bool `config:"enabled"` - Field string `config:"field"` - URLField string `config:"url_field" validate:"required"` - ValueTemplate *Template `config:"value_template"` - DateFormat string `config:"date_format"` - InitialInterval time.Duration `config:"initial_interval"` +type dateCursorConfig struct { + Enabled *bool `config:"enabled"` + Field string `config:"field"` + URLField string `config:"url_field" validate:"required"` + ValueTemplate *templateConfig `config:"value_template"` + DateFormat string `config:"date_format"` + InitialInterval time.Duration `config:"initial_interval"` } -type Template struct { +type templateConfig struct { *template.Template } -func (t *Template) Unpack(in string) error { +func (t *templateConfig) Unpack(in string) error { tpl, err := template.New("tpl").Parse(in) if err != nil { return err } - *t = Template{Template: tpl} + *t = templateConfig{Template: tpl} return nil } -type URL struct { +type urlConfig struct { *url.URL } -func (u *URL) Unpack(in string) error { +func (u *urlConfig) Unpack(in string) error { parsed, err := url.Parse(in) if err != nil { return err } - *u = URL{URL: parsed} + *u = urlConfig{URL: parsed} return nil } // IsEnabled returns true if the `enable` field is set to true in the yaml. -func (dc *DateCursor) IsEnabled() bool { +func (dc *dateCursorConfig) isEnabled() bool { return dc != nil && (dc.Enabled == nil || *dc.Enabled) } // IsEnabled returns true if the `enable` field is set to true in the yaml. -func (dc *DateCursor) GetDateFormat() string { +func (dc *dateCursorConfig) getDateFormat() string { if dc.DateFormat == "" { return time.RFC3339 } return dc.DateFormat } -func (dc *DateCursor) Validate() error { +func (dc *dateCursorConfig) Validate() error { if dc.DateFormat == "" { return nil } @@ -154,7 +154,7 @@ func (c *config) Validate() error { } } } - if c.OAuth2.IsEnabled() { + if c.OAuth2.isEnabled() { if c.APIKey != "" || c.AuthenticationScheme != "" { return errors.New("invalid configuration: oauth2 and api_key or authentication_scheme cannot be set simultaneously") } @@ -162,7 +162,7 @@ func (c *config) Validate() error { return nil } -func defaultConfig() config { +func newDefaultConfig() config { var c config c.HTTPMethod = "GET" c.HTTPClientTimeout = 60 * time.Second diff --git a/x-pack/filebeat/input/httpjson/config_oauth.go b/x-pack/filebeat/input/httpjson/config_oauth.go index 0ff55dcbc33..d7412fd0ba8 100644 --- a/x-pack/filebeat/input/httpjson/config_oauth.go +++ b/x-pack/filebeat/input/httpjson/config_oauth.go @@ -20,32 +20,32 @@ import ( "golang.org/x/oauth2/google" ) -// An OAuth2Provider represents a supported oauth provider. -type OAuth2Provider string +// An oauth2Provider represents a supported oauth provider. +type oauth2Provider string const ( - OAuth2ProviderDefault OAuth2Provider = "" // OAuth2ProviderDefault means no specific provider is set. - OAuth2ProviderAzure OAuth2Provider = "azure" // OAuth2ProviderAzure AzureAD. - OAuth2ProviderGoogle OAuth2Provider = "google" // OAuth2ProviderGoogle Google. + oauth2ProviderDefault oauth2Provider = "" // OAuth2ProviderDefault means no specific provider is set. + oauth2ProviderAzure oauth2Provider = "azure" // OAuth2ProviderAzure AzureAD. + oauth2ProviderGoogle oauth2Provider = "google" // OAuth2ProviderGoogle Google. ) -func (p *OAuth2Provider) Unpack(in string) error { - *p = OAuth2Provider(in) +func (p *oauth2Provider) Unpack(in string) error { + *p = oauth2Provider(in) return nil } -func (p OAuth2Provider) canonical() OAuth2Provider { - return OAuth2Provider(strings.ToLower(string(p))) +func (p oauth2Provider) canonical() oauth2Provider { + return oauth2Provider(strings.ToLower(string(p))) } -// OAuth2 contains information about oauth2 authentication settings. -type OAuth2 struct { +// oauth2Config contains information about oauth2 authentication settings. +type oauth2Config struct { // common oauth fields ClientID string `config:"client.id"` ClientSecret string `config:"client.secret"` Enabled *bool `config:"enabled"` EndpointParams map[string][]string `config:"endpoint_params"` - Provider OAuth2Provider `config:"provider"` + Provider oauth2Provider `config:"provider"` Scopes []string `config:"scopes"` TokenURL string `config:"token_url"` @@ -61,25 +61,26 @@ type OAuth2 struct { } // IsEnabled returns true if the `enable` field is set to true in the yaml. -func (o *OAuth2) IsEnabled() bool { +func (o *oauth2Config) isEnabled() bool { return o != nil && (o.Enabled == nil || *o.Enabled) } // Client wraps the given http.Client and returns a new one that will use the oauth authentication. -func (o *OAuth2) Client(ctx context.Context, client *http.Client) (*http.Client, error) { - ctx = context.WithValue(ctx, oauth2.HTTPClient, client) +func (o *oauth2Config) client(ctx context.Context, client *http.Client) (*http.Client, error) { + // only required to let oauth2 library to find our custom client in the context + ctx = context.WithValue(context.Background(), oauth2.HTTPClient, client) - switch o.GetProvider() { - case OAuth2ProviderAzure, OAuth2ProviderDefault: + switch o.getProvider() { + case oauth2ProviderAzure, oauth2ProviderDefault: creds := clientcredentials.Config{ ClientID: o.ClientID, ClientSecret: o.ClientSecret, - TokenURL: o.GetTokenURL(), + TokenURL: o.getTokenURL(), Scopes: o.Scopes, - EndpointParams: o.GetEndpointParams(), + EndpointParams: o.getEndpointParams(), } return creds.Client(ctx), nil - case OAuth2ProviderGoogle: + case oauth2ProviderGoogle: if o.GoogleJWTFile != "" { cfg, err := google.JWTConfigFromJSON(o.GoogleCredentialsJSON, o.Scopes...) if err != nil { @@ -100,9 +101,9 @@ func (o *OAuth2) Client(ctx context.Context, client *http.Client) (*http.Client, } // GetTokenURL returns the TokenURL. -func (o *OAuth2) GetTokenURL() string { - switch o.GetProvider() { - case OAuth2ProviderAzure: +func (o *oauth2Config) getTokenURL() string { + switch o.getProvider() { + case oauth2ProviderAzure: if o.TokenURL == "" { return endpoints.AzureAD(o.AzureTenantID).TokenURL } @@ -112,14 +113,14 @@ func (o *OAuth2) GetTokenURL() string { } // GetProvider returns provider in its canonical form. -func (o OAuth2) GetProvider() OAuth2Provider { +func (o oauth2Config) getProvider() oauth2Provider { return o.Provider.canonical() } // GetEndpointParams returns endpoint params with any provider ones combined. -func (o OAuth2) GetEndpointParams() map[string][]string { - switch o.GetProvider() { - case OAuth2ProviderAzure: +func (o oauth2Config) getEndpointParams() map[string][]string { + switch o.getProvider() { + case oauth2ProviderAzure: if o.AzureResource != "" { if o.EndpointParams == nil { o.EndpointParams = map[string][]string{} @@ -132,18 +133,18 @@ func (o OAuth2) GetEndpointParams() map[string][]string { } // Validate checks if oauth2 config is valid. -func (o *OAuth2) Validate() error { - switch o.GetProvider() { - case OAuth2ProviderAzure: +func (o *oauth2Config) Validate() error { + switch o.getProvider() { + case oauth2ProviderAzure: return o.validateAzureProvider() - case OAuth2ProviderGoogle: + case oauth2ProviderGoogle: return o.validateGoogleProvider() - case OAuth2ProviderDefault: + case oauth2ProviderDefault: if o.TokenURL == "" || o.ClientID == "" || o.ClientSecret == "" { return errors.New("invalid configuration: both token_url and client credentials must be provided") } default: - return fmt.Errorf("invalid configuration: unknown provider %q", o.GetProvider()) + return fmt.Errorf("invalid configuration: unknown provider %q", o.getProvider()) } return nil } @@ -151,7 +152,7 @@ func (o *OAuth2) Validate() error { // findDefaultGoogleCredentials will default to google.FindDefaultCredentials and will only be changed for testing purposes var findDefaultGoogleCredentials = google.FindDefaultCredentials -func (o *OAuth2) validateGoogleProvider() error { +func (o *oauth2Config) validateGoogleProvider() error { if o.TokenURL != "" || o.ClientID != "" || o.ClientSecret != "" || o.AzureTenantID != "" || o.AzureResource != "" || len(o.EndpointParams) > 0 { return errors.New("invalid configuration: none of token_url and client credentials can be used, use google.credentials_file, google.jwt_file, google.credentials_json or ADC instead") @@ -191,7 +192,7 @@ func (o *OAuth2) validateGoogleProvider() error { return fmt.Errorf("invalid configuration: no authentication credentials were configured or detected (ADC)") } -func (o *OAuth2) populateCredentialsJSONFromFile(file string) error { +func (o *oauth2Config) populateCredentialsJSONFromFile(file string) error { if _, err := os.Stat(file); os.IsNotExist(err) { return fmt.Errorf("invalid configuration: the file %q cannot be found", file) } @@ -210,7 +211,7 @@ func (o *OAuth2) populateCredentialsJSONFromFile(file string) error { return nil } -func (o *OAuth2) validateAzureProvider() error { +func (o *oauth2Config) validateAzureProvider() error { if o.TokenURL == "" && o.AzureTenantID == "" { return errors.New("invalid configuration: at least one of token_url or tenant_id must be provided") } diff --git a/x-pack/filebeat/input/httpjson/config_oauth_test.go b/x-pack/filebeat/input/httpjson/config_oauth_test.go index 3fa0eed4284..67ec63b6650 100644 --- a/x-pack/filebeat/input/httpjson/config_oauth_test.go +++ b/x-pack/filebeat/input/httpjson/config_oauth_test.go @@ -11,8 +11,8 @@ import ( func TestProviderCanonical(t *testing.T) { const ( - a OAuth2Provider = "gOoGle" - b OAuth2Provider = "google" + a oauth2Provider = "gOoGle" + b oauth2Provider = "google" ) if a.canonical() != b.canonical() { @@ -21,74 +21,74 @@ func TestProviderCanonical(t *testing.T) { } func TestGetProviderIsCanonical(t *testing.T) { - const expected OAuth2Provider = "google" + const expected oauth2Provider = "google" - oauth2 := OAuth2{Provider: "GOogle"} - if oauth2.GetProvider() != expected { + oauth2 := oauth2Config{Provider: "GOogle"} + if oauth2.getProvider() != expected { t.Fatal("GetProvider should return canonical provider") } } func TestIsEnabled(t *testing.T) { - oauth2 := OAuth2{} - if !oauth2.IsEnabled() { + oauth2 := oauth2Config{} + if !oauth2.isEnabled() { t.Fatal("OAuth2 should be enabled by default") } var enabled = false oauth2.Enabled = &enabled - if oauth2.IsEnabled() { + if oauth2.isEnabled() { t.Fatal("OAuth2 should be disabled") } enabled = true - if !oauth2.IsEnabled() { + if !oauth2.isEnabled() { t.Fatal("OAuth2 should be enabled") } } func TestGetTokenURL(t *testing.T) { const expected = "http://localhost" - oauth2 := OAuth2{TokenURL: "http://localhost"} - if got := oauth2.GetTokenURL(); got != expected { + oauth2 := oauth2Config{TokenURL: "http://localhost"} + if got := oauth2.getTokenURL(); got != expected { t.Fatalf("GetTokenURL should return the provided TokenURL but got %q", got) } } func TestGetTokenURLWithAzure(t *testing.T) { const expectedWithoutTenantID = "http://localhost" - oauth2 := OAuth2{TokenURL: "http://localhost", Provider: "azure"} - if got := oauth2.GetTokenURL(); got != expectedWithoutTenantID { + oauth2 := oauth2Config{TokenURL: "http://localhost", Provider: "azure"} + if got := oauth2.getTokenURL(); got != expectedWithoutTenantID { t.Fatalf("GetTokenURL should return the provided TokenURL but got %q", got) } oauth2.TokenURL = "" oauth2.AzureTenantID = "a_tenant_id" const expectedWithTenantID = "https://login.microsoftonline.com/a_tenant_id/oauth2/v2.0/token" - if got := oauth2.GetTokenURL(); got != expectedWithTenantID { + if got := oauth2.getTokenURL(); got != expectedWithTenantID { t.Fatalf("GetTokenURL should return the generated TokenURL but got %q", got) } } func TestGetEndpointParams(t *testing.T) { var expected = map[string][]string{"foo": {"bar"}} - oauth2 := OAuth2{EndpointParams: map[string][]string{"foo": {"bar"}}} - if got := oauth2.GetEndpointParams(); !reflect.DeepEqual(got, expected) { + oauth2 := oauth2Config{EndpointParams: map[string][]string{"foo": {"bar"}}} + if got := oauth2.getEndpointParams(); !reflect.DeepEqual(got, expected) { t.Fatalf("GetEndpointParams should return the provided EndpointParams but got %q", got) } } func TestGetEndpointParamsWithAzure(t *testing.T) { var expectedWithoutResource = map[string][]string{"foo": {"bar"}} - oauth2 := OAuth2{Provider: "azure", EndpointParams: map[string][]string{"foo": {"bar"}}} - if got := oauth2.GetEndpointParams(); !reflect.DeepEqual(got, expectedWithoutResource) { + oauth2 := oauth2Config{Provider: "azure", EndpointParams: map[string][]string{"foo": {"bar"}}} + if got := oauth2.getEndpointParams(); !reflect.DeepEqual(got, expectedWithoutResource) { t.Fatalf("GetEndpointParams should return the provided EndpointParams but got %q", got) } oauth2.AzureResource = "baz" var expectedWithResource = map[string][]string{"foo": {"bar"}, "resource": {"baz"}} - if got := oauth2.GetEndpointParams(); !reflect.DeepEqual(got, expectedWithResource) { + if got := oauth2.getEndpointParams(); !reflect.DeepEqual(got, expectedWithResource) { t.Fatalf("GetEndpointParams should return the provided EndpointParams but got %q", got) } } diff --git a/x-pack/filebeat/input/httpjson/config_test.go b/x-pack/filebeat/input/httpjson/config_test.go index 0de07311239..85c7c64848d 100644 --- a/x-pack/filebeat/input/httpjson/config_test.go +++ b/x-pack/filebeat/input/httpjson/config_test.go @@ -25,7 +25,7 @@ func TestConfigValidationCase1(t *testing.T) { "url": "localhost", } cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() + conf := newDefaultConfig() if err := cfg.Unpack(&conf); err == nil { t.Fatal("Configuration validation failed. no_http_body and http_request_body cannot coexist.") } @@ -39,7 +39,7 @@ func TestConfigValidationCase2(t *testing.T) { "url": "localhost", } cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() + conf := newDefaultConfig() if err := cfg.Unpack(&conf); err == nil { t.Fatal("Configuration validation failed. no_http_body and pagination.extra_body_content cannot coexist.") } @@ -53,7 +53,7 @@ func TestConfigValidationCase3(t *testing.T) { "url": "localhost", } cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() + conf := newDefaultConfig() if err := cfg.Unpack(&conf); err == nil { t.Fatal("Configuration validation failed. no_http_body and pagination.req_field cannot coexist.") } @@ -66,7 +66,7 @@ func TestConfigValidationCase4(t *testing.T) { "url": "localhost", } cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() + conf := newDefaultConfig() if err := cfg.Unpack(&conf); err == nil { t.Fatal("Configuration validation failed. pagination.header and pagination.req_field cannot coexist.") } @@ -79,7 +79,7 @@ func TestConfigValidationCase5(t *testing.T) { "url": "localhost", } cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() + conf := newDefaultConfig() if err := cfg.Unpack(&conf); err == nil { t.Fatal("Configuration validation failed. pagination.header and pagination.id_field cannot coexist.") } @@ -92,7 +92,7 @@ func TestConfigValidationCase6(t *testing.T) { "url": "localhost", } cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() + conf := newDefaultConfig() if err := cfg.Unpack(&conf); err == nil { t.Fatal("Configuration validation failed. pagination.header and extra_body_content cannot coexist.") } @@ -105,7 +105,7 @@ func TestConfigValidationCase7(t *testing.T) { "url": "localhost", } cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() + conf := newDefaultConfig() if err := cfg.Unpack(&conf); err == nil { t.Fatal("Configuration validation failed. http_method DELETE is not allowed.") } @@ -116,7 +116,7 @@ func TestConfigMustFailWithInvalidURL(t *testing.T) { "url": "::invalid::", } cfg := common.MustNewConfigFrom(m) - conf := defaultConfig() + conf := newDefaultConfig() err := cfg.Unpack(&conf) assert.EqualError(t, err, `parse "::invalid::": missing protocol scheme accessing 'url'`) } @@ -414,7 +414,7 @@ func TestConfigOauth2Validation(t *testing.T) { } cfg := common.MustNewConfigFrom(c.input) - conf := defaultConfig() + conf := newDefaultConfig() err := cfg.Unpack(&conf) switch { diff --git a/x-pack/filebeat/input/httpjson/date_cursor.go b/x-pack/filebeat/input/httpjson/date_cursor.go index 2a9db44bd2a..66ca659de78 100644 --- a/x-pack/filebeat/input/httpjson/date_cursor.go +++ b/x-pack/filebeat/input/httpjson/date_cursor.go @@ -7,6 +7,7 @@ package httpjson import ( "bytes" "net/url" + "text/template" "time" "github.com/elastic/beats/v7/libbeat/common" @@ -22,13 +23,12 @@ type dateCursor struct { initialInterval time.Duration dateFormat string - value string - valueTpl *Template + valueTpl *template.Template } func newDateCursorFromConfig(config config, log *logp.Logger) *dateCursor { c := &dateCursor{ - enabled: config.DateCursor.IsEnabled(), + enabled: config.DateCursor.isEnabled(), url: *config.URL.URL, } @@ -40,23 +40,23 @@ func newDateCursorFromConfig(config config, log *logp.Logger) *dateCursor { c.field = config.DateCursor.Field c.urlField = config.DateCursor.URLField c.initialInterval = config.DateCursor.InitialInterval - c.dateFormat = config.DateCursor.GetDateFormat() - c.valueTpl = config.DateCursor.ValueTemplate + c.dateFormat = config.DateCursor.getDateFormat() + c.valueTpl = config.DateCursor.ValueTemplate.Template return c } -func (c *dateCursor) getURL() string { +func (c *dateCursor) getURL(prevValue string) string { if !c.enabled { return c.url.String() } var dateStr string - if c.value == "" { + if prevValue == "" { t := timeNow().UTC().Add(-c.initialInterval) dateStr = t.Format(c.dateFormat) } else { - dateStr = c.value + dateStr = prevValue } q := c.url.Query() @@ -66,7 +66,7 @@ func (c *dateCursor) getURL() string { value = dateStr } else { buf := new(bytes.Buffer) - if err := c.valueTpl.Template.Execute(buf, dateStr); err != nil { + if err := c.valueTpl.Execute(buf, dateStr); err != nil { return c.url.String() } value = buf.String() @@ -74,32 +74,33 @@ func (c *dateCursor) getURL() string { q.Set(c.urlField, value) - c.url.RawQuery = q.Encode() + url := c.url + url.RawQuery = q.Encode() - return c.url.String() + return url.String() } -func (c *dateCursor) advance(m common.MapStr) { +func (c *dateCursor) getNextValue(m common.MapStr) string { if c.field == "" { - c.value = time.Now().UTC().Format(c.dateFormat) - return + return time.Now().UTC().Format(c.dateFormat) } v, err := m.GetValue(c.field) if err != nil { c.log.Warnf("date_cursor field: %q", err) - return + return "" } + switch t := v.(type) { case string: _, err := time.Parse(c.dateFormat, t) if err != nil { c.log.Warn("date_cursor field does not have the expected layout") - return + return "" } - c.value = t - default: - c.log.Warn("date_cursor field must be a string, cursor will not advance") - return + return t } + + c.log.Warn("date_cursor field must be a string, cursor will not advance") + return "" } diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index 766fa364864..5445197f563 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -9,12 +9,14 @@ import ( "fmt" "net" "net/http" + "net/url" "time" "github.com/hashicorp/go-retryablehttp" "go.uber.org/zap" v2 "github.com/elastic/beats/v7/filebeat/input/v2" + cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" @@ -63,30 +65,25 @@ func (log *retryLogger) Warn(format string, args ...interface{}) { log.log.Warnf(format, args...) } -type httpJSONInput struct { - config config - tlsConfig *tlscommon.TLSConfig -} - -func Plugin() v2.Plugin { +func Plugin(log *logp.Logger, store cursor.StateStore) v2.Plugin { + sim := stateless.NewInputManager(statelessConfigure) return v2.Plugin{ Name: inputName, Stability: feature.Beta, Deprecated: false, - Manager: stateless.NewInputManager(configure), - } -} - -func configure(cfg *common.Config) (stateless.Input, error) { - conf := defaultConfig() - if err := cfg.Unpack(&conf); err != nil { - return nil, err + Manager: inputManager{ + stateless: &sim, + cursor: &cursor.InputManager{ + Logger: log, + StateStore: store, + Type: inputName, + Configure: cursorConfigure, + }, + }, } - - return newHTTPJSONInput(conf) } -func newHTTPJSONInput(config config) (*httpJSONInput, error) { +func newTLSConfig(config config) (*tlscommon.TLSConfig, error) { if err := config.Validate(); err != nil { return nil, err } @@ -96,54 +93,53 @@ func newHTTPJSONInput(config config) (*httpJSONInput, error) { return nil, err } - return &httpJSONInput{ - config: config, - tlsConfig: tlsConfig, - }, nil + return tlsConfig, nil } -func (*httpJSONInput) Name() string { return inputName } - -func (in *httpJSONInput) Test(v2.TestContext) error { +func test(url *url.URL) error { port := func() string { - if in.config.URL.Port() != "" { - return in.config.URL.Port() + if url.Port() != "" { + return url.Port() } - switch in.config.URL.Scheme { + switch url.Scheme { case "https": return "443" } return "80" }() - _, err := net.DialTimeout("tcp", net.JoinHostPort(in.config.URL.Hostname(), port), time.Second) + _, err := net.DialTimeout("tcp", net.JoinHostPort(url.Hostname(), port), time.Second) if err != nil { - return fmt.Errorf("url %q is unreachable", in.config.URL) + return fmt.Errorf("url %q is unreachable", url) } return nil } -// Run starts the input and blocks until it ends the execution. -// It will return on context cancellation, any other error will be retried. -func (in *httpJSONInput) Run(ctx v2.Context, publisher stateless.Publisher) error { - log := ctx.Logger.With("url", in.config.URL) +func run( + ctx v2.Context, + config config, + tlsConfig *tlscommon.TLSConfig, + publisher cursor.Publisher, + cursor *cursor.Cursor, +) error { + log := ctx.Logger.With("url", config.URL) stdCtx := ctxtool.FromCanceller(ctx.Cancelation) - httpClient, err := in.newHTTPClient(stdCtx) + httpClient, err := newHTTPClient(stdCtx, config, tlsConfig) if err != nil { return err } - dateCursor := newDateCursorFromConfig(in.config, log) + dateCursor := newDateCursorFromConfig(config, log) - rateLimiter := newRateLimiterFromConfig(in.config, log) + rateLimiter := newRateLimiterFromConfig(config, log) - pagination := newPaginationFromConfig(in.config) + pagination := newPaginationFromConfig(config) requester := newRequester( - in.config, + config, rateLimiter, dateCursor, pagination, @@ -151,12 +147,14 @@ func (in *httpJSONInput) Run(ctx v2.Context, publisher stateless.Publisher) erro log, ) + requester.loadCursor(cursor, log) + // TODO: disallow passing interval = 0 as a mean to run once. - if in.config.Interval == 0 { + if config.Interval == 0 { return requester.processHTTPRequest(stdCtx, publisher) } - err = timed.Periodic(stdCtx, in.config.Interval, func() error { + err = timed.Periodic(stdCtx, config.Interval, func() error { log.Info("Process another repeated request.") if err := requester.processHTTPRequest(stdCtx, publisher); err != nil { log.Error(err) @@ -169,29 +167,29 @@ func (in *httpJSONInput) Run(ctx v2.Context, publisher stateless.Publisher) erro return nil } -func (in *httpJSONInput) newHTTPClient(ctx context.Context) (*http.Client, error) { +func newHTTPClient(ctx context.Context, config config, tlsConfig *tlscommon.TLSConfig) (*http.Client, error) { // Make retryable HTTP client client := &retryablehttp.Client{ HTTPClient: &http.Client{ Transport: &http.Transport{ DialContext: (&net.Dialer{ - Timeout: in.config.HTTPClientTimeout, + Timeout: config.HTTPClientTimeout, }).DialContext, - TLSClientConfig: in.tlsConfig.ToConfig(), + TLSClientConfig: tlsConfig.ToConfig(), DisableKeepAlives: true, }, - Timeout: in.config.HTTPClientTimeout, + Timeout: config.HTTPClientTimeout, }, Logger: newRetryLogger(), - RetryWaitMin: in.config.RetryWaitMin, - RetryWaitMax: in.config.RetryWaitMax, - RetryMax: in.config.RetryMax, + RetryWaitMin: config.RetryWaitMin, + RetryWaitMax: config.RetryWaitMax, + RetryMax: config.RetryMax, CheckRetry: retryablehttp.DefaultRetryPolicy, Backoff: retryablehttp.DefaultBackoff, } - if in.config.OAuth2.IsEnabled() { - return in.config.OAuth2.Client(ctx, client.StandardClient()) + if config.OAuth2.isEnabled() { + return config.OAuth2.client(ctx, client.StandardClient()) } return client.StandardClient(), nil diff --git a/x-pack/filebeat/input/httpjson/input_cursor.go b/x-pack/filebeat/input/httpjson/input_cursor.go new file mode 100644 index 00000000000..d18a91f3918 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/input_cursor.go @@ -0,0 +1,67 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package httpjson + +import ( + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" +) + +type cursorInput struct{} + +func (cursorInput) Name() string { + return "httpjson-cursor" +} + +type source struct { + config config + tlsConfig *tlscommon.TLSConfig +} + +func (src source) Name() string { + return src.config.URL.String() +} + +func cursorConfigure(cfg *common.Config) ([]cursor.Source, cursor.Input, error) { + conf := newDefaultConfig() + if err := cfg.Unpack(&conf); err != nil { + return nil, nil, err + } + return newCursorInput(conf) +} + +func newCursorInput(config config) ([]cursor.Source, cursor.Input, error) { + tlsConfig, err := newTLSConfig(config) + if err != nil { + return nil, nil, err + } + // we only allow one url per config, if we wanted to allow more than one + // each source should hold only one url + return []cursor.Source{ + &source{config: config, + tlsConfig: tlsConfig, + }, + }, + &cursorInput{}, + nil +} + +func (in *cursorInput) Test(src cursor.Source, _ v2.TestContext) error { + return test((src.(*source)).config.URL.URL) +} + +// Run starts the input and blocks until it ends the execution. +// It will return on context cancellation, any other error will be retried. +func (in *cursorInput) Run( + ctx v2.Context, + src cursor.Source, + cursor cursor.Cursor, + publisher cursor.Publisher, +) error { + s := src.(*source) + return run(ctx, s.config, s.tlsConfig, publisher, &cursor) +} diff --git a/x-pack/filebeat/input/httpjson/input_manager.go b/x-pack/filebeat/input/httpjson/input_manager.go new file mode 100644 index 00000000000..21f5066dc05 --- /dev/null +++ b/x-pack/filebeat/input/httpjson/input_manager.go @@ -0,0 +1,49 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package httpjson + +import ( + "go.uber.org/multierr" + + "github.com/elastic/go-concert/unison" + + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" + "github.com/elastic/beats/v7/libbeat/common" +) + +// inputManager wraps one stateless input manager +// and one cursor input manager. It will create one or the other +// based on the config that is passed. +type inputManager struct { + stateless *stateless.InputManager + cursor *cursor.InputManager +} + +var _ v2.InputManager = inputManager{} + +// Init initializes both wrapped input managers. +func (m inputManager) Init(grp unison.Group, mode v2.Mode) error { + return multierr.Append( + m.stateless.Init(grp, mode), + m.cursor.Init(grp, mode), + ) +} + +// Create creates a cursor input manager if the config has a date cursor set up, +// otherwise it creates a stateless input manager. +func (m inputManager) Create(cfg *common.Config) (v2.Input, error) { + var config config + if err := cfg.Unpack(&config); err != nil { + return nil, err + } + + if config.DateCursor != nil { + return m.cursor.Create(cfg) + } + + return m.stateless.Create(cfg) +} diff --git a/x-pack/filebeat/input/httpjson/input_stateless.go b/x-pack/filebeat/input/httpjson/input_stateless.go new file mode 100644 index 00000000000..c7ebf6c3d4c --- /dev/null +++ b/x-pack/filebeat/input/httpjson/input_stateless.go @@ -0,0 +1,58 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package httpjson + +import ( + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" +) + +type statelessInput struct { + config config + tlsConfig *tlscommon.TLSConfig +} + +func (statelessInput) Name() string { + return "httpjson-stateless" +} + +func statelessConfigure(cfg *common.Config) (stateless.Input, error) { + conf := newDefaultConfig() + if err := cfg.Unpack(&conf); err != nil { + return nil, err + } + return newStatelessInput(conf) +} + +func newStatelessInput(config config) (*statelessInput, error) { + tlsConfig, err := newTLSConfig(config) + if err != nil { + return nil, err + } + return &statelessInput{config: config, tlsConfig: tlsConfig}, nil +} + +func (in *statelessInput) Test(v2.TestContext) error { + return test(in.config.URL.URL) +} + +type statelessPublisher struct { + wrapped stateless.Publisher +} + +func (pub statelessPublisher) Publish(event beat.Event, _ interface{}) error { + pub.wrapped.Publish(event) + return nil +} + +// Run starts the input and blocks until it ends the execution. +// It will return on context cancellation, any other error will be retried. +func (in *statelessInput) Run(ctx v2.Context, publisher stateless.Publisher) error { + pub := statelessPublisher{wrapped: publisher} + return run(ctx, in.config, in.tlsConfig, pub, nil) +} diff --git a/x-pack/filebeat/input/httpjson/httpjson_test.go b/x-pack/filebeat/input/httpjson/input_test.go similarity index 96% rename from x-pack/filebeat/input/httpjson/httpjson_test.go rename to x-pack/filebeat/input/httpjson/input_test.go index b541c16002e..242811d2795 100644 --- a/x-pack/filebeat/input/httpjson/httpjson_test.go +++ b/x-pack/filebeat/input/httpjson/input_test.go @@ -23,7 +23,7 @@ import ( beattest "github.com/elastic/beats/v7/libbeat/publisher/testing" ) -func TestHTTPJSONInput(t *testing.T) { +func TestStatelessHTTPJSONInput(t *testing.T) { testCases := []struct { name string setupServer func(*testing.T, http.HandlerFunc, map[string]interface{}) @@ -224,20 +224,23 @@ func TestHTTPJSONInput(t *testing.T) { cfg := common.MustNewConfigFrom(tc.baseConfig) - input, err := configure(cfg) + conf := newDefaultConfig() + assert.NoError(t, cfg.Unpack(&conf)) + + input, err := newStatelessInput(conf) assert.NoError(t, err) - assert.Equal(t, "httpjson", input.Name()) + assert.Equal(t, "httpjson-stateless", input.Name()) assert.NoError(t, input.Test(v2.TestContext{})) - pub := beattest.NewChanClient(len(tc.expected)) - t.Cleanup(func() { _ = pub.Close() }) + chanClient := beattest.NewChanClient(len(tc.expected)) + t.Cleanup(func() { _ = chanClient.Close() }) ctx, cancel := newV2Context() t.Cleanup(cancel) var g errgroup.Group - g.Go(func() error { return input.Run(ctx, pub) }) + g.Go(func() error { return input.Run(ctx, chanClient) }) timeout := time.NewTimer(5 * time.Second) t.Cleanup(func() { _ = timeout.Stop() }) @@ -249,7 +252,7 @@ func TestHTTPJSONInput(t *testing.T) { case <-timeout.C: t.Errorf("timed out waiting for %d events", len(tc.expected)) return - case got := <-pub.Channel: + case got := <-chanClient.Channel: val, err := got.Fields.GetValue("message") assert.NoError(t, err) assert.JSONEq(t, tc.expected[receivedCount], val.(string)) diff --git a/x-pack/filebeat/input/httpjson/pagination.go b/x-pack/filebeat/input/httpjson/pagination.go index 9a7bf82b2b4..020bc783055 100644 --- a/x-pack/filebeat/input/httpjson/pagination.go +++ b/x-pack/filebeat/input/httpjson/pagination.go @@ -16,7 +16,7 @@ import ( type pagination struct { extraBodyContent common.MapStr - header *Header + header *headerConfig idField string requestField string urlField string @@ -24,7 +24,7 @@ type pagination struct { } func newPaginationFromConfig(config config) *pagination { - if !config.Pagination.IsEnabled() { + if !config.Pagination.isEnabled() { return nil } return &pagination{ diff --git a/x-pack/filebeat/input/httpjson/pagination_test.go b/x-pack/filebeat/input/httpjson/pagination_test.go index 9b04de75819..32e3261c1e6 100644 --- a/x-pack/filebeat/input/httpjson/pagination_test.go +++ b/x-pack/filebeat/input/httpjson/pagination_test.go @@ -42,7 +42,7 @@ func TestCreateRequestInfoFromBody(t *testing.T) { contentMap: common.MapStr{}, headers: common.MapStr{}, } - err := pagination.setRequestInfoFromBody( + _ = pagination.setRequestInfoFromBody( common.MapStr(m), common.MapStr(m), ri, diff --git a/x-pack/filebeat/input/httpjson/rate_limiter.go b/x-pack/filebeat/input/httpjson/rate_limiter.go index 57d206224ac..93c2b4a3fe7 100644 --- a/x-pack/filebeat/input/httpjson/rate_limiter.go +++ b/x-pack/filebeat/input/httpjson/rate_limiter.go @@ -122,7 +122,7 @@ func (r *rateLimiter) getRateLimit(header http.Header) (int64, error) { if err != nil { return 0, fmt.Errorf("failed to parse rate-limit reset value: %w", err) } - if time.Unix(epoch, 0).Sub(time.Now()) <= 0 { + if time.Until(time.Unix(epoch, 0)) <= 0 { return 0, nil } diff --git a/x-pack/filebeat/input/httpjson/requester.go b/x-pack/filebeat/input/httpjson/requester.go index b5f58179aa0..df0a1efb1eb 100644 --- a/x-pack/filebeat/input/httpjson/requester.go +++ b/x-pack/filebeat/input/httpjson/requester.go @@ -14,7 +14,7 @@ import ( "net/http" "strings" - stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" + cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -40,6 +40,8 @@ type requester struct { authScheme string jsonObjects string splitEventsBy string + + cursorState cursorState } func newRequester( @@ -72,9 +74,9 @@ type response struct { } // processHTTPRequest processes HTTP request, and handles pagination if enabled -func (r *requester) processHTTPRequest(ctx context.Context, publisher stateless.Publisher) error { +func (r *requester) processHTTPRequest(ctx context.Context, publisher cursor.Publisher) error { ri := &requestInfo{ - url: r.dateCursor.getURL(), + url: r.dateCursor.getURL(r.cursorState.LastDateCursorValue), contentMap: common.MapStr{}, headers: r.headers, } @@ -166,7 +168,7 @@ func (r *requester) processHTTPRequest(ctx context.Context, publisher stateless. } if lastObj != nil && r.dateCursor.enabled { - r.dateCursor.advance(common.MapStr(lastObj)) + r.updateCursorState(ri.url, r.dateCursor.getNextValue(common.MapStr(lastObj))) } return nil @@ -210,7 +212,7 @@ func (r *requester) createHTTPRequest(ctx context.Context, ri *requestInfo) (*ht } // processEventArray publishes an event for each object contained in the array. It returns the last object in the array and an error if any. -func (r *requester) processEventArray(publisher stateless.Publisher, events []interface{}) (map[string]interface{}, error) { +func (r *requester) processEventArray(publisher cursor.Publisher, events []interface{}) (map[string]interface{}, error) { var last map[string]interface{} for _, t := range events { switch v := t.(type) { @@ -221,7 +223,9 @@ func (r *requester) processEventArray(publisher stateless.Publisher, events []in if err != nil { return nil, fmt.Errorf("failed to marshal %+v: %w", e, err) } - publisher.Publish(makeEvent(string(d))) + if err := publisher.Publish(makeEvent(string(d)), r.cursorState); err != nil { + return nil, fmt.Errorf("failed to publish: %w", err) + } } default: return nil, fmt.Errorf("expected only JSON objects in the array but got a %T", v) @@ -273,3 +277,23 @@ func splitEvent(splitKey string, event map[string]interface{}) []map[string]inte return events } + +type cursorState struct { + LastCalledURL string + LastDateCursorValue string +} + +func (r *requester) updateCursorState(url, value string) { + r.cursorState.LastCalledURL = url + r.cursorState.LastDateCursorValue = value +} + +func (r *requester) loadCursor(c *cursor.Cursor, log *logp.Logger) { + if c == nil || c.IsNew() { + return + } + + if err := c.Unpack(&r.cursorState); err != nil { + log.Errorf("Reset http cursor state. Failed to read from registry: %v", err) + } +} From 2996b6f1891ec03f48ab5cf2d2b989b94e7ec1d9 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Tue, 29 Sep 2020 11:01:15 -0400 Subject: [PATCH 04/17] [Elastic Agent] Add install/uninstall sub-command (#21206) * Add install command. * Fix binary name on darwin. * Fix install of mkdir. * Update shell wrapper path for darwin. * Add ability to fix broken installation. * Fix issues with install/uninstall. * Add dedicated uninstall command. * Add enrollment at the end of install command. * Fix installation of shell wrapper. * Fix root_windows.go * Fix uninstall on Windows. * Add sleep to removal on Windows. * Fix sleep to timeout in uninstall subcommand. * Fix uninstall for nested timeout with cmd.exe. * Fix uninstall on windows to actually sleep correctly. * Fix formatting. * Add changelog. * Fixes for mage check. * Create the symlink on Windows on install, fix issue with enroll questions during install. * Refactor to remove the paths.yml and symlink dance on Windows, because new install command handles that. * Cleanup repeat GA warning. * Fix symlink on Windows. * Fix control socket on windows. * Fix socket path and authority on Windows. * Fix username matching for SYSTEM. * Fix socket path on darwin. * Fix darwin service name. Fix uninstall on systemd. * Fix install on linux. * Fix RunningInstalled on Linux. * Prevent upgrade unless conditions are correct. * Add ability to force upgradable with DEV=true when built. * Fixes from code review. * Fix issue with service description. --- NOTICE.txt | 109 +++++++++++++ dev-tools/notice/rules.json | 3 +- .../templates/linux/elastic-agent.sh.tmpl | 1 - go.mod | 3 + go.sum | 11 ++ libbeat/common/cli/input.go | 43 +++++ libbeat/common/cli/input_test.go | 63 +++++++ x-pack/elastic-agent/CHANGELOG.next.asciidoc | 1 + x-pack/elastic-agent/magefile.go | 1 + .../pkg/agent/application/paths/paths.go | 108 +++++------- .../agent/application/upgrade/step_mark.go | 54 +----- .../agent/application/upgrade/step_relink.go | 14 +- .../pkg/agent/application/upgrade/upgrade.go | 45 +++-- x-pack/elastic-agent/pkg/agent/cmd/checks.go | 57 ------- .../pkg/agent/cmd/checks_windows.go | 114 ------------- x-pack/elastic-agent/pkg/agent/cmd/common.go | 59 +------ x-pack/elastic-agent/pkg/agent/cmd/enroll.go | 50 +++++- x-pack/elastic-agent/pkg/agent/cmd/install.go | 154 ++++++++++++++++++ .../elastic-agent/pkg/agent/cmd/uninstall.go | 95 +++++++++++ .../elastic-agent/pkg/agent/control/addr.go | 7 + .../pkg/agent/control/addr_windows.go | 7 + .../pkg/agent/control/server/listener.go | 1 - .../agent/control/server/listener_windows.go | 31 +++- .../pkg/agent/install/install.go | 142 ++++++++++++++++ .../pkg/agent/install/install_unix.go | 13 ++ .../pkg/agent/install/install_windows.go | 35 ++++ .../pkg/agent/install/installed.go | 75 +++++++++ .../elastic-agent/pkg/agent/install/paths.go | 30 ++++ .../pkg/agent/install/paths_darwin.go | 29 ++++ .../pkg/agent/install/paths_windows.go | 27 +++ .../pkg/agent/install/root_unix.go | 19 +++ .../pkg/agent/install/root_windows.go | 27 +++ x-pack/elastic-agent/pkg/agent/install/svc.go | 38 +++++ .../pkg/agent/install/svc_unix.go | 15 ++ .../pkg/agent/install/svc_windows.go | 49 ++++++ .../pkg/agent/install/uninstall.go | 71 ++++++++ x-pack/elastic-agent/pkg/agent/warn/warn.go | 1 + x-pack/elastic-agent/pkg/release/upgrade.go | 10 ++ x-pack/elastic-agent/pkg/release/version.go | 17 ++ 39 files changed, 1242 insertions(+), 387 deletions(-) create mode 100644 libbeat/common/cli/input.go create mode 100644 libbeat/common/cli/input_test.go delete mode 100644 x-pack/elastic-agent/pkg/agent/cmd/checks.go delete mode 100644 x-pack/elastic-agent/pkg/agent/cmd/checks_windows.go create mode 100644 x-pack/elastic-agent/pkg/agent/cmd/install.go create mode 100644 x-pack/elastic-agent/pkg/agent/cmd/uninstall.go create mode 100644 x-pack/elastic-agent/pkg/agent/install/install.go create mode 100644 x-pack/elastic-agent/pkg/agent/install/install_unix.go create mode 100644 x-pack/elastic-agent/pkg/agent/install/install_windows.go create mode 100644 x-pack/elastic-agent/pkg/agent/install/installed.go create mode 100644 x-pack/elastic-agent/pkg/agent/install/paths.go create mode 100644 x-pack/elastic-agent/pkg/agent/install/paths_darwin.go create mode 100644 x-pack/elastic-agent/pkg/agent/install/paths_windows.go create mode 100644 x-pack/elastic-agent/pkg/agent/install/root_unix.go create mode 100644 x-pack/elastic-agent/pkg/agent/install/root_windows.go create mode 100644 x-pack/elastic-agent/pkg/agent/install/svc.go create mode 100644 x-pack/elastic-agent/pkg/agent/install/svc_unix.go create mode 100644 x-pack/elastic-agent/pkg/agent/install/svc_windows.go create mode 100644 x-pack/elastic-agent/pkg/agent/install/uninstall.go create mode 100644 x-pack/elastic-agent/pkg/release/upgrade.go diff --git a/NOTICE.txt b/NOTICE.txt index 748fe0f5e98..527c1304379 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -10907,6 +10907,36 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/blakerouse/service +Version: v1.1.1-0.20200924160513-057808572ffa +Licence type (autodetected): Zlib +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/blakerouse/service@v1.1.1-0.20200924160513-057808572ffa/LICENSE: + +Copyright (c) 2015 Daniel Theophanes + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. + + -------------------------------------------------------------------------------- Dependency : github.com/lib/pq Version: v1.1.2-0.20190507191818-2ff3cb3adc01 @@ -11843,6 +11873,37 @@ Contents of probable licence file $GOMODCACHE/github.com/oklog/ulid@v1.3.1/LICEN limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/otiai10/copy +Version: v1.2.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/otiai10/copy@v1.2.0/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2018 otiai10 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/pierrre/gotestcover Version: v0.0.0-20160517101806-924dca7d15f0 @@ -34009,6 +34070,54 @@ Contents of probable licence file $GOMODCACHE/github.com/opencontainers/runtime- limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/otiai10/curr +Version: v1.0.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/otiai10/curr@v1.0.0/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2020 Hiromu Ochiai + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/otiai10/mint +Version: v1.3.1 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/otiai10/mint@v1.3.1/LICENSE: + +Copyright 2017 otiai10 (Hiromu OCHIAI) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/oxtoacart/bpool Version: v0.0.0-20150712133111-4e1c5567d7c2 diff --git a/dev-tools/notice/rules.json b/dev-tools/notice/rules.json index 73ce763cdae..c9638a9c6cf 100644 --- a/dev-tools/notice/rules.json +++ b/dev-tools/notice/rules.json @@ -9,7 +9,8 @@ "ISC", "MIT", "MPL-2.0", - "Public Domain" + "Public Domain", + "Zlib" ], "maybelist": [ "EPL-1.0", diff --git a/dev-tools/packaging/templates/linux/elastic-agent.sh.tmpl b/dev-tools/packaging/templates/linux/elastic-agent.sh.tmpl index 744abc05702..835b5955324 100644 --- a/dev-tools/packaging/templates/linux/elastic-agent.sh.tmpl +++ b/dev-tools/packaging/templates/linux/elastic-agent.sh.tmpl @@ -6,6 +6,5 @@ exec /usr/share/{{.BeatName}}/bin/{{.BeatName}} \ --path.home /var/lib/{{.BeatName}} \ --path.config /etc/{{.BeatName}} \ - --path.data /var/lib/{{.BeatName}}/data \ --path.logs /var/log/{{.BeatName}} \ "$@" diff --git a/go.mod b/go.mod index 2fafe750879..09cd086cbee 100644 --- a/go.mod +++ b/go.mod @@ -107,6 +107,7 @@ require ( github.com/josephspurrier/goversioninfo v0.0.0-20190209210621-63e6d1acd3dd github.com/jpillora/backoff v1.0.0 // indirect github.com/jstemmer/go-junit-report v0.9.1 + github.com/kardianos/service v1.1.0 github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/lib/pq v1.1.2-0.20190507191818-2ff3cb3adc01 github.com/magefile/mage v1.10.0 @@ -123,6 +124,7 @@ require ( github.com/oklog/ulid v1.3.1 github.com/opencontainers/go-digest v1.0.0-rc1.0.20190228220655-ac19fd6e7483 // indirect github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6 // indirect + github.com/otiai10/copy v1.2.0 github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 @@ -195,6 +197,7 @@ replace ( github.com/fsnotify/fsnotify => github.com/adriansr/fsnotify v0.0.0-20180417234312-c9bbe1f46f1d github.com/google/gopacket => github.com/adriansr/gopacket v1.1.18-0.20200327165309-dd62abfa8a41 github.com/insomniacslk/dhcp => github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 // indirect + github.com/kardianos/service => github.com/blakerouse/service v1.1.1-0.20200924160513-057808572ffa github.com/tonistiigi/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c golang.org/x/tools => golang.org/x/tools v0.0.0-20200602230032-c00d67ef29d0 // release 1.14 ) diff --git a/go.sum b/go.sum index 44f6eeb2ba0..031f1faa095 100644 --- a/go.sum +++ b/go.sum @@ -127,6 +127,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blakerouse/service v1.1.1-0.20200924160513-057808572ffa h1:aXHPZwx8Y5z8r+1WPylnu095usTf6QSshaHs6nVMBc0= +github.com/blakerouse/service v1.1.1-0.20200924160513-057808572ffa/go.mod h1:RrJI2xn5vve/r32U5suTbeaSGoMU6GbNPoj36CVYcHc= github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2 h1:oMCHnXa6CCCafdPDbMh/lWRhRByN0VFLvv+g+ayx1SI= github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= @@ -563,6 +565,14 @@ github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rm github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= +github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k= +github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI= +github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.3.1 h1:BCmzIS3n71sGfHB5NMNDB3lHYPz8fWSkCAErHed//qc= +github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab5aMCs5usQRVBGThelUKBNnoSOuso= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -817,6 +827,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/libbeat/common/cli/input.go b/libbeat/common/cli/input.go new file mode 100644 index 00000000000..a6a516fd3d4 --- /dev/null +++ b/libbeat/common/cli/input.go @@ -0,0 +1,43 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cli + +import ( + "bufio" + "fmt" + "io" + "os" + + "github.com/pkg/errors" +) + +// ReadInput shows the text and ask the user to provide input. +func ReadInput(prompt string) (string, error) { + reader := bufio.NewReader(os.Stdin) + return input(reader, prompt) +} + +func input(r io.Reader, prompt string) (string, error) { + reader := bufio.NewScanner(r) + fmt.Print(prompt + " ") + + if !reader.Scan() { + return "", errors.New("error reading user input") + } + return reader.Text(), nil +} diff --git a/libbeat/common/cli/input_test.go b/libbeat/common/cli/input_test.go new file mode 100644 index 00000000000..de87b5efe2a --- /dev/null +++ b/libbeat/common/cli/input_test.go @@ -0,0 +1,63 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cli + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestReadInput(t *testing.T) { + tests := []struct { + name string + input string + res string + }{ + { + name: "Question 1?", + input: "\n", + res: "", + }, + { + name: "Question 2?", + input: "full string input\n", + res: "full string input", + }, + { + name: "Question 3?", + input: "123456789\n", + res: "123456789", + }, + { + name: "Question 4?", + input: "false\n", + res: "false", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + r := strings.NewReader(test.input) + result, err := input(r, test.name) + assert.NoError(t, err) + assert.Equal(t, test.res, result) + }) + } +} diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index 3963dda9a12..be0cfc459e8 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -26,3 +26,4 @@ - Add support for dynamic inputs with providers and `{{variable|"default"}}` substitution. {pull}20839[20839] - Add support for EQL based condition on inputs {pull}20994[20994] - Send `fleet.host.id` to Endpoint Security {pull}21042[21042] +- Add `install` and `uninstall` subcommands {pull}21206[21206] diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index 7296e8189be..4fa067f8f8b 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -658,6 +658,7 @@ func buildVars() map[string]string { if isDevFlag, devFound := os.LookupEnv(devEnv); devFound { if isDev, err := strconv.ParseBool(isDevFlag); err == nil && isDev { vars["github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release.allowEmptyPgp"] = "true" + vars["github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release.allowUpgrade"] = "true" } } diff --git a/x-pack/elastic-agent/pkg/agent/application/paths/paths.go b/x-pack/elastic-agent/pkg/agent/application/paths/paths.go index 48544ec7593..b646f3796ba 100644 --- a/x-pack/elastic-agent/pkg/agent/application/paths/paths.go +++ b/x-pack/elastic-agent/pkg/agent/application/paths/paths.go @@ -6,121 +6,87 @@ package paths import ( "flag" + "fmt" "os" "path/filepath" - "runtime" - "sync" + "strings" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" ) var ( - homePath string - configPath string - dataPath string - logsPath string - serviceName string - - overridesLoader sync.Once + topPath string + configPath string + logsPath string ) func init() { - initialHome := initialHome() + topPath = initialTop() + configPath = topPath + logsPath = topPath fs := flag.CommandLine - fs.StringVar(&homePath, "path.home", initialHome, "Agent root path") - fs.StringVar(&configPath, "path.config", initialHome, "Config path is the directory Agent looks for its config file") - fs.StringVar(&dataPath, "path.data", filepath.Join(initialHome, "data"), "Data path contains Agent managed binaries") - fs.StringVar(&logsPath, "path.logs", initialHome, "Logs path contains Agent log output") + fs.StringVar(&topPath, "path.home", topPath, "Agent root path") + fs.StringVar(&configPath, "path.config", configPath, "Config path is the directory Agent looks for its config file") + fs.StringVar(&logsPath, "path.logs", logsPath, "Logs path contains Agent log output") } -// UpdatePaths update paths based on changes in paths file. -func UpdatePaths() { - getOverrides() -} - -func getOverrides() { - type paths struct { - HomePath string `config:"path.home" yaml:"path.home"` - ConfigPath string `config:"path.config" yaml:"path.config"` - DataPath string `config:"path.data" yaml:"path.data"` - LogsPath string `config:"path.logs" yaml:"path.logs"` - ServiceName string `config:"path.service_name" yaml:"path.service_name"` - } - - defaults := &paths{ - HomePath: homePath, - ConfigPath: configPath, - DataPath: dataPath, - LogsPath: logsPath, - } - - pathsFile := filepath.Join(dataPath, "paths.yml") - rawConfig, err := config.LoadYAML(pathsFile) - if err != nil { - return - } - - rawConfig.Unpack(defaults) - homePath = defaults.HomePath - configPath = defaults.ConfigPath - dataPath = defaults.DataPath - logsPath = defaults.LogsPath - serviceName = defaults.ServiceName -} - -// ServiceName return predefined service name if defined by initial call. -func ServiceName() string { - // needs to do this at this place because otherwise it will - // get overwritten by flags behavior. - overridesLoader.Do(getOverrides) - return serviceName +// Top returns the top directory for Elastic Agent, all the versioned +// home directories live under this top-level/data/elastic-agent-${hash} +func Top() string { + return topPath } // Home returns a directory where binary lives -// Executable is not supported on nacl. func Home() string { - overridesLoader.Do(getOverrides) - return homePath + return versionedHome(topPath) } // Config returns a directory where configuration file lives func Config() string { - overridesLoader.Do(getOverrides) return configPath } // Data returns the data directory for Agent func Data() string { - overridesLoader.Do(getOverrides) - return dataPath + return filepath.Join(Top(), "data") } // Logs returns a the log directory for Agent func Logs() string { - overridesLoader.Do(getOverrides) return logsPath } +// initialTop returns the initial top-level path for the binary +// +// When nested in top-level/data/elastic-agent-${hash}/ the result is top-level/. +func initialTop() string { + exePath := retrieveExecutablePath() + if insideData(exePath) { + return filepath.Dir(filepath.Dir(exePath)) + } + return exePath +} + +// retrieveExecutablePath returns the executing binary, even if the started binary was a symlink func retrieveExecutablePath() string { execPath, err := os.Executable() if err != nil { panic(err) } - evalPath, err := filepath.EvalSymlinks(execPath) if err != nil { panic(err) } - return filepath.Dir(evalPath) } -func initialHome() string { - exePath := retrieveExecutablePath() - if runtime.GOOS == "windows" { - return exePath - } +// insideData returns true when the exePath is inside of the current Agents data path. +func insideData(exePath string) bool { + expectedPath := filepath.Join("data", fmt.Sprintf("elastic-agent-%s", release.ShortCommit())) + return strings.HasSuffix(exePath, expectedPath) +} - return filepath.Dir(filepath.Dir(exePath)) // is two level up the executable (symlink evaluated) +func versionedHome(base string) string { + return filepath.Join(base, "data", fmt.Sprintf("elastic-agent-%s", release.ShortCommit())) } diff --git a/x-pack/elastic-agent/pkg/agent/application/upgrade/step_mark.go b/x-pack/elastic-agent/pkg/agent/application/upgrade/step_mark.go index 0d8253bb9ca..53920e6ecff 100644 --- a/x-pack/elastic-agent/pkg/agent/application/upgrade/step_mark.go +++ b/x-pack/elastic-agent/pkg/agent/application/upgrade/step_mark.go @@ -6,11 +6,8 @@ package upgrade import ( "context" - "fmt" "io/ioutil" - "os" "path/filepath" - "runtime" "time" "gopkg.in/yaml.v2" @@ -41,10 +38,6 @@ type updateMarker struct { // markUpgrade marks update happened so we can handle grace period func (h *Upgrader) markUpgrade(ctx context.Context, hash string, action *fleetapi.ActionUpgrade) error { - if err := updateHomePath(hash); err != nil { - return err - } - prevVersion := release.Version() prevHash := release.Commit() if len(prevHash) > hashLen { @@ -69,55 +62,10 @@ func (h *Upgrader) markUpgrade(ctx context.Context, hash string, action *fleetap return errors.New(err, errors.TypeFilesystem, "failed to create update marker file", errors.M(errors.MetaKeyPath, markerPath)) } - activeCommitPath := filepath.Join(paths.Config(), agentCommitFile) + activeCommitPath := filepath.Join(paths.Top(), agentCommitFile) if err := ioutil.WriteFile(activeCommitPath, []byte(hash), 0644); err != nil { return errors.New(err, errors.TypeFilesystem, "failed to update active commit", errors.M(errors.MetaKeyPath, activeCommitPath)) } return nil } - -func updateHomePath(hash string) error { - if err := createPathsSymlink(hash); err != nil { - return errors.New(err, errors.TypeFilesystem, "failed to create paths symlink") - } - - pathsMap := make(map[string]string) - pathsFilepath := filepath.Join(paths.Data(), "paths.yml") - - pathsBytes, err := ioutil.ReadFile(pathsFilepath) - if err != nil { - return errors.New(err, errors.TypeConfig, "failed to read paths file") - } - - if err := yaml.Unmarshal(pathsBytes, &pathsMap); err != nil { - return errors.New(err, errors.TypeConfig, "failed to parse paths file") - } - - pathsMap["path.home"] = filepath.Join(filepath.Dir(paths.Home()), fmt.Sprintf("%s-%s", agentName, hash)) - - pathsBytes, err = yaml.Marshal(pathsMap) - if err != nil { - return errors.New(err, errors.TypeConfig, "failed to marshal paths file") - } - - return ioutil.WriteFile(pathsFilepath, pathsBytes, 0740) -} - -func createPathsSymlink(hash string) error { - // only on windows, as windows resolves PWD using symlinks in a different way. - // we create symlink for each versioned agent inside `data/` directory - // on other systems path is shared - if runtime.GOOS != "windows" { - return nil - } - - dir := filepath.Join(paths.Data(), fmt.Sprintf("%s-%s", agentName, hash)) - versionedPath := filepath.Join(dir, "data", "paths.yml") - if err := os.MkdirAll(filepath.Dir(versionedPath), 0700); err != nil { - return err - } - - pathsCfgPath := filepath.Join(paths.Data(), "paths.yml") - return os.Symlink(pathsCfgPath, versionedPath) -} diff --git a/x-pack/elastic-agent/pkg/agent/application/upgrade/step_relink.go b/x-pack/elastic-agent/pkg/agent/application/upgrade/step_relink.go index 48d22de36cf..7cbd78d4849 100644 --- a/x-pack/elastic-agent/pkg/agent/application/upgrade/step_relink.go +++ b/x-pack/elastic-agent/pkg/agent/application/upgrade/step_relink.go @@ -21,22 +21,22 @@ func (u *Upgrader) changeSymlink(ctx context.Context, newHash string) error { // create symlink to elastic-agent-{hash} hashedDir := fmt.Sprintf("%s-%s", agentName, newHash) - agentBakName := agentName + ".bak" - symlinkPath := filepath.Join(paths.Config(), agentName) - newPath := filepath.Join(paths.Data(), hashedDir, agentName) + agentPrevName := agentName + ".prev" + symlinkPath := filepath.Join(paths.Top(), agentName) + newPath := filepath.Join(paths.Top(), "data", hashedDir, agentName) // handle windows suffixes if runtime.GOOS == "windows" { - agentBakName = agentName + ".exe.back" //.bak is already used + agentPrevName = agentName + ".exe.prev" symlinkPath += ".exe" newPath += ".exe" } - bakNewPath := filepath.Join(paths.Config(), agentBakName) - if err := os.Symlink(newPath, bakNewPath); err != nil { + prevNewPath := filepath.Join(paths.Top(), agentPrevName) + if err := os.Symlink(newPath, prevNewPath); err != nil { return errors.New(err, errors.TypeFilesystem, "failed to update agent symlink") } // safely rotate - return file.SafeFileRotate(symlinkPath, bakNewPath) + return file.SafeFileRotate(symlinkPath, prevNewPath) } diff --git a/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go b/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go index 08c38aba8c5..9c2c4f02286 100644 --- a/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go +++ b/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go @@ -17,6 +17,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/install" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" @@ -32,11 +33,12 @@ const ( // Upgrader performs an upgrade type Upgrader struct { - settings *artifact.Config - log *logger.Logger - closers []context.CancelFunc - reexec reexecManager - acker acker + settings *artifact.Config + log *logger.Logger + closers []context.CancelFunc + reexec reexecManager + acker acker + upgradable bool } type reexecManager interface { @@ -51,16 +53,28 @@ type acker interface { // NewUpgrader creates an upgrader which is capable of performing upgrade operation func NewUpgrader(settings *artifact.Config, log *logger.Logger, closers []context.CancelFunc, reexec reexecManager, a acker) *Upgrader { return &Upgrader{ - settings: settings, - log: log, - closers: closers, - reexec: reexec, - acker: a, + settings: settings, + log: log, + closers: closers, + reexec: reexec, + acker: a, + upgradable: getUpgradable(), } } +// Upgradable returns true if the Elastic Agent can be upgraded. +func (u *Upgrader) Upgradable() bool { + return u.upgradable +} + // Upgrade upgrades running agent func (u *Upgrader) Upgrade(ctx context.Context, a *fleetapi.ActionUpgrade) error { + if !u.upgradable { + return fmt.Errorf( + "cannot be upgraded; must be installed with install sub-command and " + + "running under control of the systems supervisor") + } + archivePath, err := u.downloadArtifact(ctx, a.Version, a.SourceURI) if err != nil { return err @@ -134,15 +148,16 @@ func (u *Upgrader) Ack(ctx context.Context) error { return ioutil.WriteFile(markerFile, markerBytes, 0600) } -func isSubdir(base, target string) (bool, error) { - relPath, err := filepath.Rel(base, target) - return strings.HasPrefix(relPath, ".."), err -} - func rollbackInstall(hash string) { os.RemoveAll(filepath.Join(paths.Data(), fmt.Sprintf("%s-%s", agentName, hash))) } +func getUpgradable() bool { + // only upgradable if running from Agent installer and running under the + // control of the system supervisor (or built specifically with upgrading enabled) + return release.Upgradable() || (install.RunningInstalled() && install.RunningUnderSupervisor()) +} + func copyActionStore(newHash string) error { currentActionStorePath := info.AgentActionStoreFile() diff --git a/x-pack/elastic-agent/pkg/agent/cmd/checks.go b/x-pack/elastic-agent/pkg/agent/cmd/checks.go deleted file mode 100644 index 4fee7497009..00000000000 --- a/x-pack/elastic-agent/pkg/agent/cmd/checks.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -// +build !windows - -package cmd - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - - "github.com/spf13/cobra" - - // import logp flags - _ "github.com/elastic/beats/v7/libbeat/logp/configure" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" -) - -// preRunCheck is noop because -// - darwin.tar - symlink created during packaging -// - linux.tar - symlink created during packaging -// - linux.rpm - symlink created using install script -// - linux.deb - symlink created using install script -// - linux.docker - symlink created using Dockerfile -func preRunCheck(flags *globalFlags) func(cmd *cobra.Command, args []string) error { - return func(cmd *cobra.Command, args []string) error { - if sn := paths.ServiceName(); sn != "" { - // paths were created we're running as child. - return nil - } - - // get versioned path - smallHash := fmt.Sprintf("elastic-agent-%s", smallHash(release.Commit())) - commitFilepath := filepath.Join(paths.Config(), commitFile) // use other file in the future - if content, err := ioutil.ReadFile(commitFilepath); err == nil { - smallHash = hashedDirName(content) - } - - origExecPath, err := os.Executable() - if err != nil { - return err - } - reexecPath := filepath.Join(paths.Data(), smallHash, filepath.Base(origExecPath)) - - // generate paths - if err := generatePaths(filepath.Dir(reexecPath), origExecPath); err != nil { - return err - } - - paths.UpdatePaths() - return nil - } -} diff --git a/x-pack/elastic-agent/pkg/agent/cmd/checks_windows.go b/x-pack/elastic-agent/pkg/agent/cmd/checks_windows.go deleted file mode 100644 index 36108c8e08b..00000000000 --- a/x-pack/elastic-agent/pkg/agent/cmd/checks_windows.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -// +build windows - -package cmd - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - - "github.com/spf13/cobra" - - // import logp flags - _ "github.com/elastic/beats/v7/libbeat/logp/configure" - - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/reexec" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" -) - -func preRunCheck(flags *globalFlags) func(cmd *cobra.Command, args []string) error { - return func(cmd *cobra.Command, args []string) error { - if sn := paths.ServiceName(); sn != "" { - // replacing with correct service name so we - // can talk to service manager. - if !filepath.IsAbs(os.Args[0]) { - os.Args[0] = sn - } - - // paths were created we're running as child. - return nil - } - - smallHash := fmt.Sprintf("elastic-agent-%s", smallHash(release.Commit())) - commitFilepath := filepath.Join(paths.Config(), commitFile) - if content, err := ioutil.ReadFile(commitFilepath); err == nil { - smallHash = hashedDirName(content) - } - - // rename itself - origExecPath, err := os.Executable() - if err != nil { - return err - } - - if err := os.Rename(origExecPath, origExecPath+".bak"); err != nil { - return err - } - - // create symlink to elastic-agent-{hash} - reexecPath := filepath.Join(paths.Data(), smallHash, filepath.Base(origExecPath)) - if err := os.Symlink(reexecPath, origExecPath); err != nil { - return err - } - - // generate paths - if err := generatePaths(filepath.Dir(reexecPath), origExecPath); err != nil { - return err - } - - paths.UpdatePaths() - - // reexec if running run - if cmd.Use == "run" { - pathConfigFile := flags.Config() - rawConfig, err := config.LoadYAML(pathConfigFile) - if err != nil { - return errors.New(err, - fmt.Sprintf("could not read configuration file %s", pathConfigFile), - errors.TypeFilesystem, - errors.M(errors.MetaKeyPath, pathConfigFile)) - } - - cfg, err := configuration.NewFromConfig(rawConfig) - if err != nil { - return errors.New(err, - fmt.Sprintf("could not parse configuration file %s", pathConfigFile), - errors.TypeFilesystem, - errors.M(errors.MetaKeyPath, pathConfigFile)) - } - - logger, err := logger.NewFromConfig("", cfg.Settings.LoggingConfig) - if err != nil { - return err - } - - rexLogger := logger.Named("reexec") - rm := reexec.NewManager(rexLogger, reexecPath) - - argsOverrides := []string{ - "--path.data", paths.Data(), - "--path.home", filepath.Dir(reexecPath), - "--path.config", paths.Config(), - } - rm.ReExec(argsOverrides...) - - // trigger reexec - rm.ShutdownComplete() - - // return without running Run method - os.Exit(0) - } - - return nil - } -} diff --git a/x-pack/elastic-agent/pkg/agent/cmd/common.go b/x-pack/elastic-agent/pkg/agent/cmd/common.go index d5c195566bd..8ca5700f3c6 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/common.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/common.go @@ -6,15 +6,10 @@ package cmd import ( "flag" - "fmt" - "io/ioutil" "os" "path/filepath" - "runtime" - "strings" "github.com/spf13/cobra" - "gopkg.in/yaml.v2" // import logp flags _ "github.com/elastic/beats/v7/libbeat/logp/configure" @@ -58,7 +53,6 @@ func NewCommandWithArgs(args []string, streams *cli.IOStreams) *cobra.Command { // path flags cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.home")) cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.config")) - cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.data")) cmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("path.logs")) cmd.PersistentFlags().StringVarP(&flags.PathConfigFile, "c", "c", defaultConfig, `Configuration file, relative to path.config`) @@ -72,6 +66,8 @@ func NewCommandWithArgs(args []string, streams *cli.IOStreams) *cobra.Command { run := newRunCommandWithArgs(flags, args, streams) cmd.AddCommand(basecmd.NewDefaultCommandsWithArgs(args, streams)...) cmd.AddCommand(run) + cmd.AddCommand(newInstallCommandWithArgs(flags, args, streams)) + cmd.AddCommand(newUninstallCommandWithArgs(flags, args, streams)) cmd.AddCommand(newEnrollCommandWithArgs(flags, args, streams)) cmd.AddCommand(newInspectCommandWithArgs(flags, args, streams)) @@ -80,58 +76,7 @@ func NewCommandWithArgs(args []string, streams *cli.IOStreams) *cobra.Command { if reexec != nil { cmd.AddCommand(reexec) } - cmd.PersistentPreRunE = preRunCheck(flags) cmd.Run = run.Run return cmd } - -func hashedDirName(filecontent []byte) string { - s := strings.TrimSpace(string(filecontent)) - if len(s) == 0 { - return "elastic-agent" - } - - s = smallHash(s) - - return fmt.Sprintf("elastic-agent-%s", s) -} - -func smallHash(hash string) string { - if len(hash) > hashLen { - hash = hash[:hashLen] - } - - return hash -} - -func generatePaths(dir, origExec string) error { - pathsCfg := map[string]interface{}{ - "path.data": paths.Data(), - "path.home": dir, - "path.config": paths.Config(), - "path.service_name": origExec, - } - - pathsCfgPath := filepath.Join(paths.Data(), "paths.yml") - pathsContent, err := yaml.Marshal(pathsCfg) - if err != nil { - return err - } - - if err := ioutil.WriteFile(pathsCfgPath, pathsContent, 0740); err != nil { - return err - } - - if runtime.GOOS == "windows" { - // due to two binaries we need to do a path dance - // as versioned binary will look for path inside it's own directory - versionedPath := filepath.Join(dir, "data", "paths.yml") - if err := os.MkdirAll(filepath.Dir(versionedPath), 0700); err != nil { - return err - } - return os.Symlink(pathsCfgPath, versionedPath) - } - - return nil -} diff --git a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go index 6749b57b250..6a604554136 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go @@ -42,18 +42,55 @@ func newEnrollCommandWithArgs(flags *globalFlags, _ []string, streams *cli.IOStr }, } + addEnrollFlags(cmd) + cmd.Flags().BoolP("force", "f", false, "Force overwrite the current and do not prompt for confirmation") + cmd.Flags().Bool("no-restart", false, "Skip restarting the currently running daemon") + + // used by install command + cmd.Flags().BoolP("from-install", "", false, "Set by install command to signal this was executed from install") + cmd.Flags().MarkHidden("from-install") + + return cmd +} + +func addEnrollFlags(cmd *cobra.Command) { cmd.Flags().StringP("certificate-authorities", "a", "", "Comma separated list of root certificate for server verifications") cmd.Flags().StringP("ca-sha256", "p", "", "Comma separated list of certificate authorities hash pins used for certificate verifications") - cmd.Flags().BoolP("force", "f", false, "Force overwrite the current and do not prompt for confirmation") cmd.Flags().BoolP("insecure", "i", false, "Allow insecure connection to Kibana") cmd.Flags().StringP("staging", "", "", "Configures agent to download artifacts from a staging build") - cmd.Flags().Bool("no-restart", false, "Skip restarting the currently running daemon") +} - return cmd +func buildEnrollmentFlags(cmd *cobra.Command) []string { + ca, _ := cmd.Flags().GetString("certificate-authorities") + sha256, _ := cmd.Flags().GetString("ca-sha256") + insecure, _ := cmd.Flags().GetBool("insecure") + staging, _ := cmd.Flags().GetString("staging") + + args := []string{} + if ca != "" { + args = append(args, "--certificate-authorities") + args = append(args, ca) + } + if sha256 != "" { + args = append(args, "--ca-sha256") + args = append(args, sha256) + } + if insecure { + args = append(args, "--insecure") + } + if staging != "" { + args = append(args, "--staging") + args = append(args, staging) + } + return args } func enroll(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args []string) error { - warn.PrintNotGA(streams.Out) + fromInstall, _ := cmd.Flags().GetBool("from-install") + if !fromInstall { + warn.PrintNotGA(streams.Out) + } + pathConfigFile := flags.Config() rawConfig, err := application.LoadConfigFromFile(pathConfigFile) if err != nil { @@ -79,13 +116,16 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args } force, _ := cmd.Flags().GetBool("force") + if fromInstall { + force = true + } if !force { confirm, err := c.Confirm("This will replace your current settings. Do you want to continue?", true) if err != nil { return errors.New(err, "problem reading prompt response") } if !confirm { - fmt.Fprintln(streams.Out, "Enrollment was canceled by the user") + fmt.Fprintln(streams.Out, "Enrollment was cancelled by the user") return nil } } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/install.go b/x-pack/elastic-agent/pkg/agent/cmd/install.go new file mode 100644 index 00000000000..177f79912c9 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/cmd/install.go @@ -0,0 +1,154 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cmd + +import ( + "fmt" + "os" + "os/exec" + + "github.com/spf13/cobra" + + c "github.com/elastic/beats/v7/libbeat/common/cli" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/install" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/warn" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli" +) + +func newInstallCommandWithArgs(flags *globalFlags, _ []string, streams *cli.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "install", + Short: "Install Elastic Agent permanently on this system", + Long: `This will install Elastic Agent permanently on this system and will become managed by the systems service manager. + +Unless all the require command-line parameters are provided or -f is used this command will ask questions on how you +would like the Agent to operate. +`, + Run: func(c *cobra.Command, args []string) { + if err := installCmd(streams, c, flags, args); err != nil { + fmt.Fprintf(streams.Err, "%v\n", err) + os.Exit(1) + } + }, + } + + cmd.Flags().StringP("kibana-url", "k", "", "URL of Kibana to enroll Agent into Fleet") + cmd.Flags().StringP("enrollment-token", "t", "", "Enrollment token to use to enroll Agent into Fleet") + cmd.Flags().BoolP("force", "f", false, "Force overwrite the current and do not prompt for confirmation") + addEnrollFlags(cmd) + + return cmd +} + +func installCmd(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args []string) error { + if !install.HasRoot() { + return fmt.Errorf("unable to perform install command, not executed with %s permissions", install.PermissionUser) + } + status, reason := install.Status() + if status == install.Installed { + return fmt.Errorf("already installed at: %s", install.InstallPath) + } + + warn.PrintNotGA(streams.Out) + force, _ := cmd.Flags().GetBool("force") + if status == install.Broken { + if !force { + fmt.Fprintf(streams.Out, "Elastic Agent is installed but currently broken: %s\n", reason) + confirm, err := c.Confirm(fmt.Sprintf("Continuing will re-install Elastic Agent over the current installation at %s. Do you want to continue?", install.InstallPath), true) + if err != nil { + return fmt.Errorf("Error: problem reading prompt response") + } + if !confirm { + return fmt.Errorf("installation was cancelled by the user") + } + } + } else { + if !force { + confirm, err := c.Confirm(fmt.Sprintf("Elastic Agent will be installed at %s and will run as a service. Do you want to continue?", install.InstallPath), true) + if err != nil { + return fmt.Errorf("Error: problem reading prompt response") + } + if !confirm { + return fmt.Errorf("installation was cancelled by the user") + } + } + } + + err := install.Install() + if err != nil { + return fmt.Errorf("Error: %s", err) + } + err = install.StartService() + if err != nil { + fmt.Fprintf(streams.Out, "Installation of required system files was successful, but starting of the service failed.\n") + return err + } + fmt.Fprintf(streams.Out, "Installation was successful and Elastic Agent is running.\n") + + askEnroll := true + kibana, _ := cmd.Flags().GetString("kibana-url") + token, _ := cmd.Flags().GetString("enrollment-token") + if kibana != "" && token != "" { + askEnroll = false + } + if force { + askEnroll = false + } + if askEnroll { + confirm, err := c.Confirm("Do you want to enroll this Agent into Fleet?", true) + if err != nil { + return fmt.Errorf("problem reading prompt response") + } + if !confirm { + // not enrolling, all done (standalone mode) + return nil + } + } + if !askEnroll && (kibana == "" || token == "") { + // force was performed without required enrollment arguments, all done (standalone mode) + return nil + } + + if kibana == "" { + kibana, err = c.ReadInput("Kibana URL you want to enroll this Agent into:") + if err != nil { + return fmt.Errorf("problem reading prompt response") + } + if kibana == "" { + fmt.Fprintf(streams.Out, "Enrollment cancelled because no URL was provided.\n") + return nil + } + } + if token == "" { + token, err = c.ReadInput("Fleet enrollment token:") + if err != nil { + return fmt.Errorf("problem reading prompt response") + } + if token == "" { + fmt.Fprintf(streams.Out, "Enrollment cancelled because no enrollment token was provided.\n") + return nil + } + } + + enrollArgs := []string{"enroll", kibana, token, "--from-install"} + enrollArgs = append(enrollArgs, buildEnrollmentFlags(cmd)...) + enrollCmd := exec.Command(install.ExecutablePath(), enrollArgs...) + enrollCmd.Stdin = os.Stdin + enrollCmd.Stdout = os.Stdout + enrollCmd.Stderr = os.Stderr + err = enrollCmd.Start() + if err != nil { + return fmt.Errorf("failed to execute enroll command: %s", err) + } + err = enrollCmd.Wait() + if err == nil { + return nil + } + exitErr, ok := err.(*exec.ExitError) + if ok { + return fmt.Errorf("enroll command failed with exit code: %d", exitErr.ExitCode()) + } + return fmt.Errorf("enroll command failed for unknown reason: %s", err) +} diff --git a/x-pack/elastic-agent/pkg/agent/cmd/uninstall.go b/x-pack/elastic-agent/pkg/agent/cmd/uninstall.go new file mode 100644 index 00000000000..d215a15e337 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/cmd/uninstall.go @@ -0,0 +1,95 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cmd + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + + "github.com/spf13/cobra" + + c "github.com/elastic/beats/v7/libbeat/common/cli" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/install" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli" +) + +func newUninstallCommandWithArgs(flags *globalFlags, _ []string, streams *cli.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "uninstall", + Short: "Uninstall permanent Elastic Agent from this system", + Long: `This will uninstall permanent Elastic Agent from this system and will no longer be managed by this system. + +Unless -f is used this command will ask confirmation before performing removal. +`, + Run: func(c *cobra.Command, args []string) { + if err := uninstallCmd(streams, c, flags, args); err != nil { + fmt.Fprintf(streams.Err, "%v\n", err) + os.Exit(1) + } + }, + } + + cmd.Flags().BoolP("force", "f", false, "Force overwrite the current and do not prompt for confirmation") + + return cmd +} + +func uninstallCmd(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args []string) error { + if !install.HasRoot() { + return fmt.Errorf("unable to perform uninstall command, not executed with %s permissions", install.PermissionUser) + } + status, reason := install.Status() + if status == install.NotInstalled { + return fmt.Errorf("not installed") + } + if status == install.Installed && !install.RunningInstalled() { + return fmt.Errorf("can only be uninstall by executing the installed Elastic Agent at: %s", install.ExecutablePath()) + } + + force, _ := cmd.Flags().GetBool("force") + if status == install.Broken { + if !force { + fmt.Fprintf(streams.Out, "Elastic Agent is installed but currently broken: %s\n", reason) + confirm, err := c.Confirm(fmt.Sprintf("Continuing will uninstall the broken Elastic Agent at %s. Do you want to continue?", install.InstallPath), true) + if err != nil { + return fmt.Errorf("problem reading prompt response") + } + if !confirm { + return fmt.Errorf("uninstall was cancelled by the user") + } + } + } else { + if !force { + confirm, err := c.Confirm(fmt.Sprintf("Elastic Agent will be uninstalled from your system at %s. Do you want to continue?", install.InstallPath), true) + if err != nil { + return fmt.Errorf("problem reading prompt response") + } + if !confirm { + return fmt.Errorf("uninstall was cancelled by the user") + } + } + } + + err := install.Uninstall() + if err != nil { + return err + } + fmt.Fprintf(streams.Out, "Elastic Agent has been uninstalled.\n") + + if runtime.GOOS == "windows" { + // The installation path will still exists because we are executing from that + // directory. So cmd.exe is spawned that sleeps for 2 seconds (using ping, recommend way from + // from Windows) then rmdir is performed. + rmdir := exec.Command( + filepath.Join(os.Getenv("windir"), "system32", "cmd.exe"), + "/C", "ping", "-n", "2", "127.0.0.1", "&&", "rmdir", "/s", "/q", install.InstallPath) + _ = rmdir.Start() + } + + return nil +} diff --git a/x-pack/elastic-agent/pkg/agent/control/addr.go b/x-pack/elastic-agent/pkg/agent/control/addr.go index 20bc1e6a005..31005e8e34d 100644 --- a/x-pack/elastic-agent/pkg/agent/control/addr.go +++ b/x-pack/elastic-agent/pkg/agent/control/addr.go @@ -11,10 +11,17 @@ import ( "fmt" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/install" ) // Address returns the address to connect to Elastic Agent daemon. func Address() string { + // when installed the control address is fixed + if install.RunningInstalled() { + return install.SocketPath + } + + // not install, adjust the path based on data path data := paths.Data() // entire string cannot be longer than 107 characters, this forces the // length to always be 88 characters (but unique per data path) diff --git a/x-pack/elastic-agent/pkg/agent/control/addr_windows.go b/x-pack/elastic-agent/pkg/agent/control/addr_windows.go index bf2e164fbae..cbfcdf2c99e 100644 --- a/x-pack/elastic-agent/pkg/agent/control/addr_windows.go +++ b/x-pack/elastic-agent/pkg/agent/control/addr_windows.go @@ -11,10 +11,17 @@ import ( "fmt" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/install" ) // Address returns the address to connect to Elastic Agent daemon. func Address() string { + // when installed the control address is fixed + if install.RunningInstalled() { + return install.SocketPath + } + + // not install, adjust the path based on data path data := paths.Data() // entire string cannot be longer than 256 characters, this forces the // length to always be 87 characters (but unique per data path) diff --git a/x-pack/elastic-agent/pkg/agent/control/server/listener.go b/x-pack/elastic-agent/pkg/agent/control/server/listener.go index 7edfc7b8ee9..3090f3d140d 100644 --- a/x-pack/elastic-agent/pkg/agent/control/server/listener.go +++ b/x-pack/elastic-agent/pkg/agent/control/server/listener.go @@ -8,7 +8,6 @@ package server import ( "fmt" - "net" "os" "path/filepath" diff --git a/x-pack/elastic-agent/pkg/agent/control/server/listener_windows.go b/x-pack/elastic-agent/pkg/agent/control/server/listener_windows.go index f98c32bcee3..eaedc9f88f2 100644 --- a/x-pack/elastic-agent/pkg/agent/control/server/listener_windows.go +++ b/x-pack/elastic-agent/pkg/agent/control/server/listener_windows.go @@ -10,6 +10,8 @@ import ( "net" "os/user" + "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/api/npipe" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control" @@ -18,11 +20,7 @@ import ( // createListener creates a named pipe listener on Windows func createListener(_ *logger.Logger) (net.Listener, error) { - u, err := user.Current() - if err != nil { - return nil, err - } - sd, err := npipe.DefaultSD(u.Username) + sd, err := securityDescriptor() if err != nil { return nil, err } @@ -32,3 +30,26 @@ func createListener(_ *logger.Logger) (net.Listener, error) { func cleanupListener(_ *logger.Logger) { // nothing to do on windows } + +func securityDescriptor() (string, error) { + u, err := user.Current() + if err != nil { + return "", errors.Wrap(err, "failed to get current user") + } + // Named pipe security and access rights. + // We create the pipe and the specific users should only be able to write to it. + // See docs: https://docs.microsoft.com/en-us/windows/win32/ipc/named-pipe-security-and-access-rights + // String definition: https://docs.microsoft.com/en-us/windows/win32/secauthz/ace-strings + // Give generic read/write access to the specified user. + descriptor := "D:P(A;;GA;;;" + u.Uid + ")" + if u.Username == "NT AUTHORITY\\SYSTEM" { + // running as SYSTEM, include Administrators group so Administrators can talk over + // the named pipe to the running Elastic Agent system process + admin, err := user.LookupGroup("Administrators") + if err != nil { + return "", errors.Wrap(err, "failed to lookup Administrators group") + } + descriptor += "(A;;GA;;;" + admin.Gid + ")" + } + return descriptor, nil +} diff --git a/x-pack/elastic-agent/pkg/agent/install/install.go b/x-pack/elastic-agent/pkg/agent/install/install.go new file mode 100644 index 00000000000..2705ea0bfd9 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/install/install.go @@ -0,0 +1,142 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package install + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/otiai10/copy" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" +) + +// Install installs Elastic Agent persistently on the system including creating and starting its service. +func Install() error { + dir, err := findDirectory() + if err != nil { + return errors.New(err, "failed to discover the source directory for installation", errors.TypeFilesystem) + } + + // uninstall current installation + err = Uninstall() + if err != nil { + return err + } + + // ensure parent directory exists, copy source into install path + err = os.MkdirAll(filepath.Dir(InstallPath), 0755) + if err != nil { + return errors.New( + err, + fmt.Sprintf("failed to create installation parent directory (%s)", filepath.Dir(InstallPath)), + errors.M("directory", filepath.Dir(InstallPath))) + } + err = copy.Copy(dir, InstallPath, copy.Options{ + OnSymlink: func(_ string) copy.SymlinkAction { + return copy.Shallow + }, + Sync: true, + }) + if err != nil { + return errors.New( + err, + fmt.Sprintf("failed to copy source directory (%s) to destination (%s)", dir, InstallPath), + errors.M("source", dir), errors.M("destination", InstallPath)) + } + + // place shell wrapper, if present on platform + if ShellWrapperPath != "" { + err = ioutil.WriteFile(ShellWrapperPath, []byte(ShellWrapper), 0755) + if err != nil { + return errors.New( + err, + fmt.Sprintf("failed to write shell wrapper (%s)", ShellWrapperPath), + errors.M("destination", ShellWrapperPath)) + } + } + + // post install (per platform) + err = postInstall() + if err != nil { + return err + } + + // install service + svc, err := newService() + if err != nil { + return err + } + err = svc.Install() + if err != nil { + return errors.New( + err, + fmt.Sprintf("failed to install service (%s)", ServiceName), + errors.M("service", ServiceName)) + } + return nil +} + +// StartService starts the installed service. +// +// This should only be called after Install is successful. +func StartService() error { + svc, err := newService() + if err != nil { + return err + } + err = svc.Start() + if err != nil { + return errors.New( + err, + fmt.Sprintf("failed to start service (%s)", ServiceName), + errors.M("service", ServiceName)) + } + return nil +} + +// findDirectory returns the directory to copy into the installation location. +// +// This also verifies that the discovered directory is a valid directory for installation. +func findDirectory() (string, error) { + execPath, err := os.Executable() + if err != nil { + return "", err + } + execPath, err = filepath.Abs(execPath) + if err != nil { + return "", err + } + sourceDir := filepath.Dir(execPath) + if insideData(sourceDir) { + // executable path is being reported as being down inside of data path + // move up to directories to perform the copy + sourceDir = filepath.Dir(filepath.Dir(sourceDir)) + } + err = verifyDirectory(sourceDir) + if err != nil { + return "", err + } + return sourceDir, nil +} + +// verifyDirectory ensures that the directory includes the executable. +func verifyDirectory(dir string) error { + _, err := os.Stat(filepath.Join(dir, BinaryName)) + if os.IsNotExist(err) { + return fmt.Errorf("missing %s", BinaryName) + } + return nil +} + +// insideData returns true when the exePath is inside of the current Agents data path. +func insideData(exePath string) bool { + expectedPath := filepath.Join("data", fmt.Sprintf("elastic-agent-%s", release.ShortCommit())) + return strings.HasSuffix(exePath, expectedPath) +} diff --git a/x-pack/elastic-agent/pkg/agent/install/install_unix.go b/x-pack/elastic-agent/pkg/agent/install/install_unix.go new file mode 100644 index 00000000000..07d696d580a --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/install/install_unix.go @@ -0,0 +1,13 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !windows + +package install + +// postInstall performs post installation for unix-based systems. +func postInstall() error { + // do nothing + return nil +} diff --git a/x-pack/elastic-agent/pkg/agent/install/install_windows.go b/x-pack/elastic-agent/pkg/agent/install/install_windows.go new file mode 100644 index 00000000000..ec10467ce79 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/install/install_windows.go @@ -0,0 +1,35 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build windows + +package install + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" +) + +// postInstall performs post installation for Windows systems. +func postInstall() error { + // delete the top-level elastic-agent.exe + binary := filepath.Join(InstallPath, BinaryName) + err := os.Remove(binary) + if err != nil { + // do not handle does not exist, it should have existed + return err + } + + // create top-level symlink to nested binary + realBinary := filepath.Join(InstallPath, "data", fmt.Sprintf("elastic-agent-%s", release.ShortCommit()), BinaryName) + err = os.Symlink(realBinary, binary) + if err != nil { + return err + } + + return nil +} diff --git a/x-pack/elastic-agent/pkg/agent/install/installed.go b/x-pack/elastic-agent/pkg/agent/install/installed.go new file mode 100644 index 00000000000..9f294078242 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/install/installed.go @@ -0,0 +1,75 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package install + +import ( + "os" + "path/filepath" + + "github.com/kardianos/service" +) + +// StatusType is the return status types. +type StatusType int + +const ( + // NotInstalled returned when Elastic Agent is not installed. + NotInstalled StatusType = iota + // Installed returned when Elastic Agent is installed currectly. + Installed + // Broken returned when Elastic Agent is installed but broken. + Broken +) + +// Status returns the installation status of Agent. +func Status() (StatusType, string) { + expected := filepath.Join(InstallPath, BinaryName) + status, reason := checkService() + _, err := os.Stat(expected) + if os.IsNotExist(err) { + if status == Installed { + // service installed, but no install path + return Broken, "service exists but installation path is missing" + } + return NotInstalled, "no install path or service" + } + if status == NotInstalled { + // install path present, but not service + return Broken, reason + } + return Installed, "" +} + +// RunningInstalled returns true when executing Agent is the installed Agent. +// +// This verifies the running executable path based on hard-coded paths +// for each platform type. +func RunningInstalled() bool { + expected := filepath.Join(InstallPath, BinaryName) + execPath, _ := os.Executable() + execPath, _ = filepath.Abs(execPath) + execName := filepath.Base(execPath) + execDir := filepath.Dir(execPath) + if insideData(execDir) { + // executable path is being reported as being down inside of data path + // move up to directories to perform the comparison + execDir = filepath.Dir(filepath.Dir(execDir)) + execPath = filepath.Join(execDir, execName) + } + return expected == execPath +} + +// checkService only checks the status of the service. +func checkService() (StatusType, string) { + svc, err := newService() + if err != nil { + return NotInstalled, "unable to check service status" + } + status, _ := svc.Status() + if status == service.StatusUnknown { + return NotInstalled, "service is not installed" + } + return Installed, "" +} diff --git a/x-pack/elastic-agent/pkg/agent/install/paths.go b/x-pack/elastic-agent/pkg/agent/install/paths.go new file mode 100644 index 00000000000..5936c31926a --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/install/paths.go @@ -0,0 +1,30 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !darwin +// +build !windows + +package install + +const ( + // BinaryName is the name of the installed binary. + BinaryName = "elastic-agent" + + // InstallPath is the installation path using for install command. + InstallPath = "/opt/Elastic/Agent" + + // SocketPath is the socket path used when installed. + SocketPath = "unix:///run/elastic-agent.sock" + + // ServiceName is the service name when installed. + ServiceName = "elastic-agent" + + // ShellWrapperPath is the path to the installed shell wrapper. + ShellWrapperPath = "/usr/bin/elastic-agent" + + // ShellWrapper is the wrapper that is installed. + ShellWrapper = `#!/bin/sh +exec /opt/Elastic/Agent/elastic-agent $@ +` +) diff --git a/x-pack/elastic-agent/pkg/agent/install/paths_darwin.go b/x-pack/elastic-agent/pkg/agent/install/paths_darwin.go new file mode 100644 index 00000000000..f6ebcdfdbae --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/install/paths_darwin.go @@ -0,0 +1,29 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build darwin + +package install + +const ( + // BinaryName is the name of the installed binary. + BinaryName = "elastic-agent" + + // InstallPath is the installation path using for install command. + InstallPath = "/Library/Elastic/Agent" + + // SocketPath is the socket path used when installed. + SocketPath = "unix:///var/run/elastic-agent.sock" + + // ServiceName is the service name when installed. + ServiceName = "co.elastic.elastic-agent" + + // ShellWrapperPath is the path to the installed shell wrapper. + ShellWrapperPath = "/usr/local/bin/elastic-agent" + + // ShellWrapper is the wrapper that is installed. + ShellWrapper = `#!/bin/sh +exec /Library/Elastic/Agent/elastic-agent $@ +` +) diff --git a/x-pack/elastic-agent/pkg/agent/install/paths_windows.go b/x-pack/elastic-agent/pkg/agent/install/paths_windows.go new file mode 100644 index 00000000000..3af4c79d941 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/install/paths_windows.go @@ -0,0 +1,27 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build windows + +package install + +const ( + // BinaryName is the name of the installed binary. + BinaryName = "elastic-agent.exe" + + // InstallPath is the installation path using for install command. + InstallPath = `C:\Program Files\Elastic\Agent` + + // SocketPath is the socket path used when installed. + SocketPath = `\\.\pipe\elastic-agent-system` + + // ServiceName is the service name when installed. + ServiceName = "Elastic Agent" + + // ShellWrapperPath is the path to the installed shell wrapper. + ShellWrapperPath = "" // no wrapper on Windows + + // ShellWrapper is the wrapper that is installed. + ShellWrapper = "" // no wrapper on Windows +) diff --git a/x-pack/elastic-agent/pkg/agent/install/root_unix.go b/x-pack/elastic-agent/pkg/agent/install/root_unix.go new file mode 100644 index 00000000000..b79f104d98d --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/install/root_unix.go @@ -0,0 +1,19 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !windows + +package install + +import "os" + +const ( + // PermissionUser is the permission level the user needs to be. + PermissionUser = "root" +) + +// HasRoot returns true if the user has root permissions. +func HasRoot() bool { + return os.Getegid() == 0 +} diff --git a/x-pack/elastic-agent/pkg/agent/install/root_windows.go b/x-pack/elastic-agent/pkg/agent/install/root_windows.go new file mode 100644 index 00000000000..c72432084ac --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/install/root_windows.go @@ -0,0 +1,27 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build windows + +package install + +import ( + "os" +) + +const ( + // PermissionUser is the permission level the user needs to be. + PermissionUser = "Administrator" +) + +// HasRoot returns true if the user has Administrator/SYSTEM permissions. +func HasRoot() bool { + // only valid rights can open the physical drive + f, err := os.Open("\\\\.\\PHYSICALDRIVE0") + if err != nil { + return false + } + defer f.Close() + return true +} diff --git a/x-pack/elastic-agent/pkg/agent/install/svc.go b/x-pack/elastic-agent/pkg/agent/install/svc.go new file mode 100644 index 00000000000..18cbc6d840f --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/install/svc.go @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package install + +import ( + "path/filepath" + + "github.com/kardianos/service" +) + +const ( + // ServiceDisplayName is the service display name for the service. + ServiceDisplayName = "Elastic Agent" + + // ServiceDescription is the description for the service. + ServiceDescription = "Elastic Agent is a unified agent to observe, monitor and protect your system." +) + +// ExecutablePath returns the path for the installed Agents executable. +func ExecutablePath() string { + exec := filepath.Join(InstallPath, BinaryName) + if ShellWrapperPath != "" { + exec = ShellWrapperPath + } + return exec +} + +func newService() (service.Service, error) { + return service.New(nil, &service.Config{ + Name: ServiceName, + DisplayName: ServiceDisplayName, + Description: ServiceDescription, + Executable: ExecutablePath(), + WorkingDirectory: InstallPath, + }) +} diff --git a/x-pack/elastic-agent/pkg/agent/install/svc_unix.go b/x-pack/elastic-agent/pkg/agent/install/svc_unix.go new file mode 100644 index 00000000000..c7acb998489 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/install/svc_unix.go @@ -0,0 +1,15 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !windows + +package install + +import "os" + +// RunningUnderSupervisor returns true when executing Agent is running under +// the supervisor processes of the OS. +func RunningUnderSupervisor() bool { + return os.Getppid() == 1 +} diff --git a/x-pack/elastic-agent/pkg/agent/install/svc_windows.go b/x-pack/elastic-agent/pkg/agent/install/svc_windows.go new file mode 100644 index 00000000000..9084f3b5ea7 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/install/svc_windows.go @@ -0,0 +1,49 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build windows + +package install + +import ( + "golang.org/x/sys/windows" +) + +// RunningUnderSupervisor returns true when executing Agent is running under +// the supervisor processes of the OS. +func RunningUnderSupervisor() bool { + serviceSid, err := allocSid(windows.SECURITY_SERVICE_RID) + if err != nil { + return false + } + defer windows.FreeSid(serviceSid) + + t, err := windows.OpenCurrentProcessToken() + if err != nil { + return false + } + defer t.Close() + + gs, err := t.GetTokenGroups() + if err != nil { + return false + } + + for _, g := range gs.AllGroups() { + if windows.EqualSid(g.Sid, serviceSid) { + return true + } + } + return false +} + +func allocSid(subAuth0 uint32) (*windows.SID, error) { + var sid *windows.SID + err := windows.AllocateAndInitializeSid(&windows.SECURITY_NT_AUTHORITY, + 1, subAuth0, 0, 0, 0, 0, 0, 0, 0, &sid) + if err != nil { + return nil, err + } + return sid, nil +} diff --git a/x-pack/elastic-agent/pkg/agent/install/uninstall.go b/x-pack/elastic-agent/pkg/agent/install/uninstall.go new file mode 100644 index 00000000000..381427eb8c7 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/install/uninstall.go @@ -0,0 +1,71 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package install + +import ( + "fmt" + "os" + "runtime" + + "github.com/kardianos/service" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" +) + +// Uninstall uninstalls persistently Elastic Agent on the system. +func Uninstall() error { + // uninstall the current service + svc, err := newService() + if err != nil { + return err + } + status, _ := svc.Status() + if status == service.StatusRunning { + err := svc.Stop() + if err != nil { + return errors.New( + err, + fmt.Sprintf("failed to stop service (%s)", ServiceName), + errors.M("service", ServiceName)) + } + status = service.StatusStopped + } + if status == service.StatusStopped { + err := svc.Uninstall() + if err != nil { + return errors.New( + err, + fmt.Sprintf("failed to uninstall service (%s)", ServiceName), + errors.M("service", ServiceName)) + } + } + + // remove, if present on platform + if ShellWrapperPath != "" { + err = os.Remove(ShellWrapperPath) + if !os.IsNotExist(err) && err != nil { + return errors.New( + err, + fmt.Sprintf("failed to remove shell wrapper (%s)", ShellWrapperPath), + errors.M("destination", ShellWrapperPath)) + } + } + + // remove existing directory + err = os.RemoveAll(InstallPath) + if err != nil { + if runtime.GOOS == "windows" { + // possible to fail on Windows, because elastic-agent.exe is running from + // this directory. + return nil + } + return errors.New( + err, + fmt.Sprintf("failed to remove installation directory (%s)", InstallPath), + errors.M("directory", InstallPath)) + } + + return nil +} diff --git a/x-pack/elastic-agent/pkg/agent/warn/warn.go b/x-pack/elastic-agent/pkg/agent/warn/warn.go index 03d746992f6..182d2af50b0 100644 --- a/x-pack/elastic-agent/pkg/agent/warn/warn.go +++ b/x-pack/elastic-agent/pkg/agent/warn/warn.go @@ -21,4 +21,5 @@ func LogNotGA(log *logger.Logger) { // PrintNotGA writes to the received writer that the Agent is not GA. func PrintNotGA(output io.Writer) { fmt.Fprintln(output, message) + fmt.Fprintln(output) } diff --git a/x-pack/elastic-agent/pkg/release/upgrade.go b/x-pack/elastic-agent/pkg/release/upgrade.go new file mode 100644 index 00000000000..ac1e8552dd2 --- /dev/null +++ b/x-pack/elastic-agent/pkg/release/upgrade.go @@ -0,0 +1,10 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package release + +// Upgradable return true when release is built specifically for upgrading. +func Upgradable() bool { + return allowUpgrade == "true" +} diff --git a/x-pack/elastic-agent/pkg/release/version.go b/x-pack/elastic-agent/pkg/release/version.go index 37579ac86de..dae609a6733 100644 --- a/x-pack/elastic-agent/pkg/release/version.go +++ b/x-pack/elastic-agent/pkg/release/version.go @@ -12,6 +12,10 @@ import ( libbeatVersion "github.com/elastic/beats/v7/libbeat/version" ) +const ( + hashLen = 6 +) + // snapshot is a flag marking build as a snapshot. var snapshot = "" @@ -19,11 +23,24 @@ var snapshot = "" // without valid pgp var allowEmptyPgp string +// allowUpgrade is used as a debug flag and allows working +// with upgrade without requiring Agent to be installed correctly +var allowUpgrade string + // Commit returns the current build hash or unknown if it was not injected in the build process. func Commit() string { return libbeatVersion.Commit() } +// ShortCommit returns commit up to 6 characters. +func ShortCommit() string { + hash := Commit() + if len(hash) > hashLen { + hash = hash[:hashLen] + } + return hash +} + // BuildTime returns the build time of the binaries. func BuildTime() time.Time { return libbeatVersion.BuildTime() From c9827a8511ac80f1859855c1296ef63c6d2f00f9 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Tue, 29 Sep 2020 11:20:12 -0400 Subject: [PATCH 05/17] [Elastic Agent] Rename *ConfigChange to PolicyChange (#20779) [Elastic Agent] Rename *ConfigChange to PolicyChange (#20779) Co-authored-by: Michal Pristas --- x-pack/elastic-agent/CHANGELOG.next.asciidoc | 1 + .../cmd/fakewebapi/action_example.json | 4 +-- .../pkg/agent/application/action_store.go | 18 +++++------ .../agent/application/action_store_test.go | 14 ++++---- .../agent/application/fleet_gateway_test.go | 4 +-- .../handler_action_policy_change.go | 14 ++++---- .../handler_action_policy_change_test.go | 32 +++++++++---------- .../agent/application/inspect_config_cmd.go | 4 +-- .../pkg/agent/application/managed_mode.go | 4 +-- .../agent/application/managed_mode_test.go | 8 ++--- .../pkg/fleetapi/ack_cmd_test.go | 6 ++-- x-pack/elastic-agent/pkg/fleetapi/action.go | 22 ++++++------- .../pkg/fleetapi/checkin_cmd_test.go | 12 +++---- 13 files changed, 72 insertions(+), 71 deletions(-) diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index be0cfc459e8..2ba08864ae8 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -10,6 +10,7 @@ - Docker container is not run as root by default. {pull}21213[21213] ==== Bugfixes +- Fix rename *ConfigChange to *PolicyChange to align on changes in the UI. {pull}20779[20779] - Thread safe sorted set {pull}21290[21290] - Copy Action store on upgrade {pull}21298[21298] - Include inputs in action store actions {pull}21298[21298] diff --git a/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/action_example.json b/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/action_example.json index 327b79ed347..f08e98942ea 100644 --- a/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/action_example.json +++ b/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/action_example.json @@ -2,9 +2,9 @@ "action": "checkin", "actions": [ { - "type": "CONFIG_CHANGE", + "type": "POLICY_CHANGE", "data": { - "config": { + "policy": { "id": "default", "outputs": { "default": { diff --git a/x-pack/elastic-agent/pkg/agent/application/action_store.go b/x-pack/elastic-agent/pkg/agent/application/action_store.go index 25dbf7a5b82..ce4ea785cf7 100644 --- a/x-pack/elastic-agent/pkg/agent/application/action_store.go +++ b/x-pack/elastic-agent/pkg/agent/application/action_store.go @@ -35,7 +35,7 @@ func newActionStore(log *logger.Logger, store storeLoad) (*actionStore, error) { } defer reader.Close() - var action actionConfigChangeSerializer + var action ActionPolicyChangeSerializer dec := yaml.NewDecoder(reader) err = dec.Decode(&action) @@ -49,7 +49,7 @@ func newActionStore(log *logger.Logger, store storeLoad) (*actionStore, error) { return nil, err } - apc := fleetapi.ActionConfigChange(action) + apc := fleetapi.ActionPolicyChange(action) return &actionStore{ log: log, @@ -62,7 +62,7 @@ func newActionStore(log *logger.Logger, store storeLoad) (*actionStore, error) { // any other type of action will be silently ignored. func (s *actionStore) Add(a action) { switch v := a.(type) { - case *fleetapi.ActionConfigChange, *fleetapi.ActionUnenroll: + case *fleetapi.ActionPolicyChange, *fleetapi.ActionUnenroll: // Only persist the action if the action is different. if s.action != nil && s.action.ID() == v.ID() { return @@ -79,8 +79,8 @@ func (s *actionStore) Save() error { } var reader io.Reader - if apc, ok := s.action.(*fleetapi.ActionConfigChange); ok { - serialize := actionConfigChangeSerializer(*apc) + if apc, ok := s.action.(*fleetapi.ActionPolicyChange); ok { + serialize := ActionPolicyChangeSerializer(*apc) r, err := yamlToReader(&serialize) if err != nil { @@ -120,7 +120,7 @@ func (s *actionStore) Actions() []action { return []action{s.action} } -// actionConfigChangeSerializer is a struct that adds a YAML serialization, I don't think serialization +// ActionPolicyChangeSerializer is a struct that adds a YAML serialization, I don't think serialization // is a concern of the fleetapi package. I went this route so I don't have to do much refactoring. // // There are four ways to achieve the same results: @@ -130,14 +130,14 @@ func (s *actionStore) Actions() []action { // 4. We have two sets of type. // // This could be done in a refactoring. -type actionConfigChangeSerializer struct { +type ActionPolicyChangeSerializer struct { ActionID string `yaml:"action_id"` ActionType string `yaml:"action_type"` - Config map[string]interface{} `yaml:"config"` + Policy map[string]interface{} `yaml:"policy"` } // Add a guards between the serializer structs and the original struct. -var _ actionConfigChangeSerializer = actionConfigChangeSerializer(fleetapi.ActionConfigChange{}) +var _ ActionPolicyChangeSerializer = ActionPolicyChangeSerializer(fleetapi.ActionPolicyChange{}) // actionUnenrollSerializer is a struct that adds a YAML serialization, type actionUnenrollSerializer struct { diff --git a/x-pack/elastic-agent/pkg/agent/application/action_store_test.go b/x-pack/elastic-agent/pkg/agent/application/action_store_test.go index 4205deda8b6..f2691d66db6 100644 --- a/x-pack/elastic-agent/pkg/agent/application/action_store_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/action_store_test.go @@ -57,10 +57,10 @@ func TestActionStore(t *testing.T) { t.Run("can save to disk known action type", withFile(func(t *testing.T, file string) { - actionConfigChange := &fleetapi.ActionConfigChange{ + ActionPolicyChange := &fleetapi.ActionPolicyChange{ ActionID: "abc123", - ActionType: "CONFIG_CHANGE", - Config: map[string]interface{}{ + ActionType: "POLICY_CHANGE", + Policy: map[string]interface{}{ "hello": "world", }, } @@ -70,7 +70,7 @@ func TestActionStore(t *testing.T) { require.NoError(t, err) require.Equal(t, 0, len(store.Actions())) - store.Add(actionConfigChange) + store.Add(ActionPolicyChange) err = store.Save() require.NoError(t, err) require.Equal(t, 1, len(store.Actions())) @@ -82,12 +82,12 @@ func TestActionStore(t *testing.T) { actions := store1.Actions() require.Equal(t, 1, len(actions)) - require.Equal(t, actionConfigChange, actions[0]) + require.Equal(t, ActionPolicyChange, actions[0]) })) t.Run("when we ACK we save to disk", withFile(func(t *testing.T, file string) { - actionConfigChange := &fleetapi.ActionConfigChange{ + ActionPolicyChange := &fleetapi.ActionPolicyChange{ ActionID: "abc123", } @@ -98,7 +98,7 @@ func TestActionStore(t *testing.T) { acker := newActionStoreAcker(&testAcker{}, store) require.Equal(t, 0, len(store.Actions())) - require.NoError(t, acker.Ack(context.Background(), actionConfigChange)) + require.NoError(t, acker.Ack(context.Background(), ActionPolicyChange)) require.Equal(t, 1, len(store.Actions())) })) } diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go index bd9037416dc..cfcd1f46994 100644 --- a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go @@ -208,10 +208,10 @@ func TestFleetGateway(t *testing.T) { { "actions": [ { - "type": "CONFIG_CHANGE", + "type": "POLICY_CHANGE", "id": "id1", "data": { - "config": { + "policy": { "id": "policy-id" } } diff --git a/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change.go b/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change.go index 34fd5716980..81dc1444816 100644 --- a/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change.go +++ b/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change.go @@ -13,24 +13,24 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" ) -type handlerConfigChange struct { +type handlerPolicyChange struct { log *logger.Logger emitter emitterFunc } -func (h *handlerConfigChange) Handle(ctx context.Context, a action, acker fleetAcker) error { - h.log.Debugf("handlerConfigChange: action '%+v' received", a) - action, ok := a.(*fleetapi.ActionConfigChange) +func (h *handlerPolicyChange) Handle(ctx context.Context, a action, acker fleetAcker) error { + h.log.Debugf("handlerPolicyChange: action '%+v' received", a) + action, ok := a.(*fleetapi.ActionPolicyChange) if !ok { - return fmt.Errorf("invalid type, expected ActionConfigChange and received %T", a) + return fmt.Errorf("invalid type, expected ActionPolicyChange and received %T", a) } - c, err := LoadConfig(action.Config) + c, err := LoadConfig(action.Policy) if err != nil { return errors.New(err, "could not parse the configuration from the policy", errors.TypeConfig) } - h.log.Debugf("handlerConfigChange: emit configuration for action %+v", a) + h.log.Debugf("handlerPolicyChange: emit configuration for action %+v", a) if err := h.emitter(c); err != nil { return err } diff --git a/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change_test.go b/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change_test.go index b95c259e7c7..ce4802b68e6 100644 --- a/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change_test.go @@ -36,13 +36,13 @@ func TestPolicyChange(t *testing.T) { emitter := &mockEmitter{} conf := map[string]interface{}{"hello": "world"} - action := &fleetapi.ActionConfigChange{ + action := &fleetapi.ActionPolicyChange{ ActionID: "abc123", - ActionType: "CONFIG_CHANGE", - Config: conf, + ActionType: "POLICY_CHANGE", + Policy: conf, } - handler := &handlerConfigChange{log: log, emitter: emitter.Emitter} + handler := &handlerPolicyChange{log: log, emitter: emitter.Emitter} err := handler.Handle(context.Background(), action, ack) require.NoError(t, err) @@ -54,13 +54,13 @@ func TestPolicyChange(t *testing.T) { emitter := &mockEmitter{err: mockErr} conf := map[string]interface{}{"hello": "world"} - action := &fleetapi.ActionConfigChange{ + action := &fleetapi.ActionPolicyChange{ ActionID: "abc123", - ActionType: "CONFIG_CHANGE", - Config: conf, + ActionType: "POLICY_CHANGE", + Policy: conf, } - handler := &handlerConfigChange{log: log, emitter: emitter.Emitter} + handler := &handlerPolicyChange{log: log, emitter: emitter.Emitter} err := handler.Handle(context.Background(), action, ack) require.Error(t, err) @@ -77,13 +77,13 @@ func TestPolicyAcked(t *testing.T) { config := map[string]interface{}{"hello": "world"} actionID := "abc123" - action := &fleetapi.ActionConfigChange{ + action := &fleetapi.ActionPolicyChange{ ActionID: actionID, - ActionType: "CONFIG_CHANGE", - Config: config, + ActionType: "POLICY_CHANGE", + Policy: config, } - handler := &handlerConfigChange{log: log, emitter: emitter.Emitter} + handler := &handlerPolicyChange{log: log, emitter: emitter.Emitter} err := handler.Handle(context.Background(), action, tacker) require.Error(t, err) @@ -99,13 +99,13 @@ func TestPolicyAcked(t *testing.T) { config := map[string]interface{}{"hello": "world"} actionID := "abc123" - action := &fleetapi.ActionConfigChange{ + action := &fleetapi.ActionPolicyChange{ ActionID: actionID, - ActionType: "CONFIG_CHANGE", - Config: config, + ActionType: "POLICY_CHANGE", + Policy: config, } - handler := &handlerConfigChange{log: log, emitter: emitter.Emitter} + handler := &handlerPolicyChange{log: log, emitter: emitter.Emitter} err := handler.Handle(context.Background(), action, tacker) require.NoError(t, err) diff --git a/x-pack/elastic-agent/pkg/agent/application/inspect_config_cmd.go b/x-pack/elastic-agent/pkg/agent/application/inspect_config_cmd.go index 2c53fc62bf2..edf1ad8cdf2 100644 --- a/x-pack/elastic-agent/pkg/agent/application/inspect_config_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/application/inspect_config_cmd.go @@ -106,13 +106,13 @@ func loadFleetConfig(cfg *config.Config) (map[string]interface{}, error) { } for _, c := range as.Actions() { - cfgChange, ok := c.(*fleetapi.ActionConfigChange) + cfgChange, ok := c.(*fleetapi.ActionPolicyChange) if !ok { continue } fmt.Println("Action ID:", cfgChange.ID()) - return cfgChange.Config, nil + return cfgChange.Policy, nil } return nil, nil } diff --git a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go index a4e4bf92379..12a9c242780 100644 --- a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go +++ b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go @@ -207,8 +207,8 @@ func newManaged( acker) actionDispatcher.MustRegister( - &fleetapi.ActionConfigChange{}, - &handlerConfigChange{ + &fleetapi.ActionPolicyChange{}, + &handlerPolicyChange{ log: log, emitter: emit, }, diff --git a/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go b/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go index 9b51016a126..81f2419f936 100644 --- a/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go @@ -40,8 +40,8 @@ func TestManagedModeRouting(t *testing.T) { require.NoError(t, err) actionDispatcher.MustRegister( - &fleetapi.ActionConfigChange{}, - &handlerConfigChange{ + &fleetapi.ActionPolicyChange{}, + &handlerPolicyChange{ log: log, emitter: emit, }, @@ -100,9 +100,9 @@ const fleetResponse = ` "action": "checkin", "actions": [{ "agent_id": "17e93530-7f42-11ea-9330-71e968b29fa4", - "type": "CONFIG_CHANGE", + "type": "POLICY_CHANGE", "data": { - "config": { + "policy": { "id": "86561d50-7f3b-11ea-9fab-3db3bdb4efa4", "outputs": { "default": { diff --git a/x-pack/elastic-agent/pkg/fleetapi/ack_cmd_test.go b/x-pack/elastic-agent/pkg/fleetapi/ack_cmd_test.go index a9e3aebc25b..75940e928b7 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/ack_cmd_test.go +++ b/x-pack/elastic-agent/pkg/fleetapi/ack_cmd_test.go @@ -46,10 +46,10 @@ func TestAck(t *testing.T) { return mux }, withAPIKey, func(t *testing.T, client clienter) { - action := &ActionConfigChange{ + action := &ActionPolicyChange{ ActionID: "my-id", - ActionType: "CONFIG_CHANGE", - Config: map[string]interface{}{ + ActionType: "POLICY_CHANGE", + Policy: map[string]interface{}{ "id": "config_id", }, } diff --git a/x-pack/elastic-agent/pkg/fleetapi/action.go b/x-pack/elastic-agent/pkg/fleetapi/action.go index efb4e1672aa..d53b7fdfcfb 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/action.go +++ b/x-pack/elastic-agent/pkg/fleetapi/action.go @@ -17,8 +17,8 @@ const ( ActionTypeUpgrade = "UPGRADE" // ActionTypeUnenroll specifies unenroll action. ActionTypeUnenroll = "UNENROLL" - // ActionTypeConfigChange specifies config change action. - ActionTypeConfigChange = "CONFIG_CHANGE" + // ActionTypePolicyChange specifies policy change action. + ActionTypePolicyChange = "POLICY_CHANGE" ) // Action base interface for all the implemented action from the fleet API. @@ -66,14 +66,14 @@ func (a *ActionUnknown) OriginalType() string { return a.originalType } -// ActionConfigChange is a request to apply a new -type ActionConfigChange struct { +// ActionPolicyChange is a request to apply a new +type ActionPolicyChange struct { ActionID string ActionType string - Config map[string]interface{} `json:"config"` + Policy map[string]interface{} `json:"policy"` } -func (a *ActionConfigChange) String() string { +func (a *ActionPolicyChange) String() string { var s strings.Builder s.WriteString("action_id: ") s.WriteString(a.ActionID) @@ -83,12 +83,12 @@ func (a *ActionConfigChange) String() string { } // Type returns the type of the Action. -func (a *ActionConfigChange) Type() string { +func (a *ActionPolicyChange) Type() string { return a.ActionType } // ID returns the ID of the Action. -func (a *ActionConfigChange) ID() string { +func (a *ActionPolicyChange) ID() string { return a.ActionID } @@ -169,14 +169,14 @@ func (a *Actions) UnmarshalJSON(data []byte) error { for _, response := range responses { switch response.ActionType { - case ActionTypeConfigChange: - action = &ActionConfigChange{ + case ActionTypePolicyChange: + action = &ActionPolicyChange{ ActionID: response.ActionID, ActionType: response.ActionType, } if err := json.Unmarshal(response.Data, action); err != nil { return errors.New(err, - "fail to decode CONFIG_CHANGE action", + "fail to decode POLICY_CHANGE action", errors.TypeConfig) } case ActionTypeUnenroll: diff --git a/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd_test.go b/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd_test.go index 953b86a260e..af8b4da81ea 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd_test.go +++ b/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd_test.go @@ -56,10 +56,10 @@ func TestCheckin(t *testing.T) { raw := ` { "actions": [{ - "type": "CONFIG_CHANGE", + "type": "POLICY_CHANGE", "id": "id1", "data": { - "config": { + "policy": { "id": "policy-id", "outputs": { "default": { @@ -102,7 +102,7 @@ func TestCheckin(t *testing.T) { // ActionPolicyChange require.Equal(t, "id1", r.Actions[0].ID()) - require.Equal(t, "CONFIG_CHANGE", r.Actions[0].Type()) + require.Equal(t, "POLICY_CHANGE", r.Actions[0].Type()) }, )) @@ -112,10 +112,10 @@ func TestCheckin(t *testing.T) { { "actions": [ { - "type": "CONFIG_CHANGE", + "type": "POLICY_CHANGE", "id": "id1", "data": { - "config": { + "policy": { "id": "policy-id", "outputs": { "default": { @@ -163,7 +163,7 @@ func TestCheckin(t *testing.T) { // ActionPolicyChange require.Equal(t, "id1", r.Actions[0].ID()) - require.Equal(t, "CONFIG_CHANGE", r.Actions[0].Type()) + require.Equal(t, "POLICY_CHANGE", r.Actions[0].Type()) // UnknownAction require.Equal(t, "id2", r.Actions[1].ID()) From a29dcc27cf8965d20b7919b01ef3f14288515653 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 29 Sep 2020 09:23:36 -0600 Subject: [PATCH 06/17] Fix debug message to show actual SQS message ID (#20614) --- x-pack/filebeat/input/s3/input.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/s3/input.go index 1085c9dccbd..6b57789b48f 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/s3/input.go @@ -288,7 +288,7 @@ func (p *s3Input) processorKeepAlive(svcSQS sqsiface.ClientAPI, message sqs.Mess // When ACK done, message will be deleted. Or when message is // not s3 ObjectCreated event related(handleSQSMessage function // failed), it will be removed as well. - p.logger.Debug("Deleting message from SQS: ", message.MessageId) + p.logger.Debug("Deleting message from SQS: ", *message.MessageId) // only delete sqs message when errC is closed with no error err := p.deleteMessage(queueURL, *message.ReceiptHandle, svcSQS) if err != nil { From e6c3d89234a449f671a0a68ccba2a224b0e4a31f Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 29 Sep 2020 18:40:26 +0200 Subject: [PATCH 07/17] Add UBI 8 image to the dependencies report (#21374) Red Hat certification process requires to include the UBI image in the list of dependencies for the project. Append it to the current list of dependencies. This requires to include a new sourceURL column in the CSV, because UBI images have a different url and source url. This is kept the same for existing dependencies. --- dev-tools/dependencies-report | 11 +++++++++++ dev-tools/notice/dependencies.csv.tmpl | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/dev-tools/dependencies-report b/dev-tools/dependencies-report index a824d0f4bf9..a2662a4ab9a 100755 --- a/dev-tools/dependencies-report +++ b/dev-tools/dependencies-report @@ -41,3 +41,14 @@ go list -m -json all $@ | go run go.elastic.co/go-licence-detector \ -noticeTemplate "$SRCPATH/notice/dependencies.csv.tmpl" \ -noticeOut "$outfile" \ -depsOut "" + + +# Fill-in required values for UBI images +# Check headers in $SRCPATH/notice/dependencies.csv.tmpl: +# name,url,version,revision,license +ubi8url='https://catalog.redhat.com/software/containers/ubi8/ubi-minimal/5c359a62bed8bd75a2c3fba8' +ubi8source='https://oss-dependencies.elastic.co/redhat/ubi/ubi-minimal-8-source.tar.gz' +ubilicense='Custom;https://www.redhat.com/licenses/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf,https://oss-dependencies.elastic.co/redhat/ubi/ubi-minimal-8-source.tar.gz' +cat <> $outfile +Red Hat Universal Base Image,$ubi8url,8,,$ubilicense,$ubi8source +EOF diff --git a/dev-tools/notice/dependencies.csv.tmpl b/dev-tools/notice/dependencies.csv.tmpl index 6ab2a970199..088314378f6 100644 --- a/dev-tools/notice/dependencies.csv.tmpl +++ b/dev-tools/notice/dependencies.csv.tmpl @@ -1,7 +1,7 @@ {{- define "depInfo" -}} {{- range $i, $dep := . }} -{{ $dep.Name }},{{ $dep.URL }},{{ $dep.Version | canonicalVersion }},{{ $dep.Version | revision }},{{ $dep.LicenceType }} +{{ $dep.Name }},{{ $dep.URL }},{{ $dep.Version | canonicalVersion }},{{ $dep.Version | revision }},{{ $dep.LicenceType }},{{ $dep.URL }} {{- end -}} {{- end -}} -name,url,version,revision,license{{ template "depInfo" .Direct }}{{ template "depInfo" .Indirect }} +name,url,version,revision,license,sourceURL{{ template "depInfo" .Direct }}{{ template "depInfo" .Indirect }} From bb84f3f778eb8fd1f23fc5d88d118506c421b355 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Tue, 29 Sep 2020 12:51:55 -0400 Subject: [PATCH 08/17] [libbeat] Fix position writing in the disk queue --- libbeat/publisher/queue/diskqueue/state_file.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/libbeat/publisher/queue/diskqueue/state_file.go b/libbeat/publisher/queue/diskqueue/state_file.go index 2ff14e3e5e2..d8cbb5690ac 100644 --- a/libbeat/publisher/queue/diskqueue/state_file.go +++ b/libbeat/publisher/queue/diskqueue/state_file.go @@ -82,12 +82,14 @@ func writeQueuePositionToHandle( } // Want to write: version (0), segment id, segment offset. - elems := []interface{}{uint32(0), position.segmentID, position.offset} - for _, elem := range elems { - err = binary.Write(file, binary.LittleEndian, &elem) - if err != nil { - return err - } + err = binary.Write(file, binary.LittleEndian, uint32(0)) + if err != nil { + return err + } + err = binary.Write(file, binary.LittleEndian, position.segmentID) + if err != nil { + return err } - return nil + err = binary.Write(file, binary.LittleEndian, position.offset) + return err } From d8d35a16b0beae02cb8048df2a66c3a743d0633f Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Tue, 29 Sep 2020 13:00:27 -0400 Subject: [PATCH 09/17] Fix shutdown tracking in s3 input (#21380) --- x-pack/filebeat/input/s3/input.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/s3/input.go index 6b57789b48f..83dc48428ee 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/s3/input.go @@ -190,11 +190,11 @@ func (p *s3Input) Run() { p.workerWg.Add(1) go p.run(svcSQS, svcS3, visibilityTimeout) - p.workerWg.Done() }) } func (p *s3Input) run(svcSQS sqsiface.ClientAPI, svcS3 s3iface.ClientAPI, visibilityTimeout int64) { + defer p.workerWg.Done() defer p.logger.Infof("s3 input worker for '%v' has stopped.", p.config.QueueURL) p.logger.Infof("s3 input worker has started. with queueURL: %v", p.config.QueueURL) From 7a369caadc1557699e8b86b6e2d1150e76d554b5 Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Tue, 29 Sep 2020 11:21:33 -0700 Subject: [PATCH 10/17] docs: fix apt/yum formatting (#21362) --- libbeat/docs/repositories.asciidoc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libbeat/docs/repositories.asciidoc b/libbeat/docs/repositories.asciidoc index a7104414465..1b27a6c0b44 100644 --- a/libbeat/docs/repositories.asciidoc +++ b/libbeat/docs/repositories.asciidoc @@ -122,7 +122,7 @@ sudo apt-get update && sudo apt-get install {beatname_pkg} -------------------------------------------------- sudo systemctl enable {beatname_pkg} -------------------------------------------------- - ++ If your system does not use `systemd` then run: + ["source","sh",subs="attributes"] @@ -224,7 +224,7 @@ sudo yum install {beatname_pkg} -------------------------------------------------- sudo systemctl enable {beatname_pkg} -------------------------------------------------- - ++ If your system does not use `systemd` then run: + ["source","sh",subs="attributes"] @@ -233,4 +233,3 @@ sudo chkconfig --add {beatname_pkg} -------------------------------------------------- endif::[] - From ef6274d0d1e36308a333cbed69846a1bd63528ae Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Tue, 29 Sep 2020 16:13:38 -0400 Subject: [PATCH 11/17] [libbeat] Enable WriteAheadLimit in the disk queue (#21391) --- libbeat/publisher/queue/diskqueue/config.go | 12 ++++++-- .../publisher/queue/diskqueue/core_loop.go | 28 ++++++++++++------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/libbeat/publisher/queue/diskqueue/config.go b/libbeat/publisher/queue/diskqueue/config.go index f39f608361d..6a165a665db 100644 --- a/libbeat/publisher/queue/diskqueue/config.go +++ b/libbeat/publisher/queue/diskqueue/config.go @@ -106,8 +106,8 @@ func DefaultSettings() Settings { MaxSegmentSize: 100 * (1 << 20), // 100MiB MaxBufferSize: (1 << 30), // 1GiB - ReadAheadLimit: 256, - WriteAheadLimit: 1024, + ReadAheadLimit: 512, + WriteAheadLimit: 2048, } } @@ -129,6 +129,14 @@ func SettingsForUserConfig(config *common.Config) (Settings, error) { // divided by 10. settings.MaxSegmentSize = uint64(userConfig.MaxSize) / 10 } + + if userConfig.ReadAheadLimit != nil { + settings.ReadAheadLimit = *userConfig.ReadAheadLimit + } + if userConfig.WriteAheadLimit != nil { + settings.WriteAheadLimit = *userConfig.WriteAheadLimit + } + return settings, nil } diff --git a/libbeat/publisher/queue/diskqueue/core_loop.go b/libbeat/publisher/queue/diskqueue/core_loop.go index 56a50b5a422..638d9da2f40 100644 --- a/libbeat/publisher/queue/diskqueue/core_loop.go +++ b/libbeat/publisher/queue/diskqueue/core_loop.go @@ -58,10 +58,15 @@ func (dq *diskQueue) run() { // The writer loop completed a request, so check if there is more // data to be sent. dq.maybeWritePending() - // We also check whether the reader loop is waiting for the data - // that was just written. + + // The data that was just written is now available for reading, so check + // if we should start a new read request. dq.maybeReadPending() + // pendingFrames should now be empty. If any producers were blocked + // because pendingFrames hit settings.WriteAheadLimit, wake them up. + dq.maybeUnblockProducers() + // Reader loop handling case readerLoopResponse := <-dq.readerLoop.responseChan: dq.handleReaderLoopResponse(readerLoopResponse) @@ -417,22 +422,25 @@ func (dq *diskQueue) enqueueWriteFrame(frame *writeFrame) { }) } -// canAcceptFrameOfSize checks whether there is enough free space in the -// queue (subject to settings.MaxBufferSize) to accept a new frame with -// the given size. Size includes both the serialized data and the frame -// header / footer; the easy way to do this for a writeFrame is to pass +// canAcceptFrameOfSize checks whether there is enough free space in the queue +// (subject to settings.{MaxBufferSize, WriteAheadLimit}) to accept a new +// frame with the given size. Size includes both the serialized data and the +// frame header / footer; the easy way to do this for a writeFrame is to pass // in frame.sizeOnDisk(). // Capacity calculations do not include requests in the blockedProducers // list (that data is owned by its callers and we can't touch it until // we are ready to respond). That allows this helper to be used both while // handling producer requests and while deciding whether to unblock // producers after free capacity increases. -// If we decide to add limits on how many events / bytes can be stored -// in pendingFrames (to avoid unbounded memory use if the input is faster -// than the disk), this is the function to modify. func (dq *diskQueue) canAcceptFrameOfSize(frameSize uint64) bool { + // If pendingFrames is already at the WriteAheadLimit, we can't accept + // any new frames right now. + if len(dq.pendingFrames) >= dq.settings.WriteAheadLimit { + return false + } + + // If the queue size is unbounded (max == 0), we accept. if dq.settings.MaxBufferSize == 0 { - // Currently we impose no limitations if the queue size is unbounded. return true } From cd28ee52a701df6148ae7f452c38c0b2a4842662 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 30 Sep 2020 13:28:42 +0200 Subject: [PATCH 12/17] Move Kubernetes events metricset to its own block in reference config (#21407) In the reference configuration the event metricset was in the block of configurations for kube-state-metrics. --- deploy/kubernetes/metricbeat-kubernetes.yaml | 6 ++++-- .../metricbeat/metricbeat-daemonset-configmap.yaml | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml index 4bc1976e40e..32d1010f4d0 100644 --- a/deploy/kubernetes/metricbeat-kubernetes.yaml +++ b/deploy/kubernetes/metricbeat-kubernetes.yaml @@ -36,8 +36,6 @@ data: - state_cronjob - state_resourcequota - state_statefulset - # Uncomment this to get k8s events: - #- event - module: kubernetes metricsets: - apiserver @@ -46,6 +44,10 @@ data: ssl.certificate_authorities: - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt period: 30s + # Uncomment this to get k8s events: + #- module: kubernetes + # metricsets: + # - event # To enable hints based autodiscover uncomment this: #- type: kubernetes # node: ${NODE_NAME} diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml index 98fffb86ad0..b95bf478765 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml @@ -36,8 +36,6 @@ data: - state_cronjob - state_resourcequota - state_statefulset - # Uncomment this to get k8s events: - #- event - module: kubernetes metricsets: - apiserver @@ -46,6 +44,10 @@ data: ssl.certificate_authorities: - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt period: 30s + # Uncomment this to get k8s events: + #- module: kubernetes + # metricsets: + # - event # To enable hints based autodiscover uncomment this: #- type: kubernetes # node: ${NODE_NAME} From 222b069bf496b70d7378a58a5cf756de01f60d0c Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Wed, 30 Sep 2020 15:14:15 +0200 Subject: [PATCH 13/17] fix: use a fixed version of setuptools (#21393) --- libbeat/tests/system/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index 08843d6144d..86071d5dd65 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -37,6 +37,7 @@ PyYAML==5.3.1 redis==2.10.6 requests==2.20.0 semver==2.8.1 +setuptools==47.3.2 six==1.15.0 stomp.py==4.1.22 termcolor==1.1.0 From 3deda877fbbc7c136bacf00b2e055f5b3093f594 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Wed, 30 Sep 2020 15:14:48 +0200 Subject: [PATCH 14/17] fix: mark flaky tests (#21300) * fix: mark flaky test * Apply suggestions from code review * Update libbeat/outputs/elasticsearch/client_integration_test.go * Apply suggestions from code review * Apply suggestions from code review --- auditbeat/module/file_integrity/eventreader_test.go | 2 ++ filebeat/tests/system/test_reload_inputs.py | 3 +++ x-pack/auditbeat/tests/system/test_metricsets.py | 3 +++ 3 files changed, 8 insertions(+) diff --git a/auditbeat/module/file_integrity/eventreader_test.go b/auditbeat/module/file_integrity/eventreader_test.go index 6d9b02f7867..a00367f350a 100644 --- a/auditbeat/module/file_integrity/eventreader_test.go +++ b/auditbeat/module/file_integrity/eventreader_test.go @@ -41,6 +41,7 @@ func init() { const ErrorSharingViolation syscall.Errno = 32 func TestEventReader(t *testing.T) { + t.Skip("Flaky test: about 1/10 of bulds fails https://github.com/elastic/beats/issues/21302") // Make dir to monitor. dir, err := ioutil.TempDir("", "audit") if err != nil { @@ -240,6 +241,7 @@ func TestEventReader(t *testing.T) { } func TestRaces(t *testing.T) { + t.Skip("Flaky test: about 1/20 of bulds fails https://github.com/elastic/beats/issues/21303") const ( fileMode os.FileMode = 0640 N = 100 diff --git a/filebeat/tests/system/test_reload_inputs.py b/filebeat/tests/system/test_reload_inputs.py index 4de554fe2c6..4c5fa1abfba 100644 --- a/filebeat/tests/system/test_reload_inputs.py +++ b/filebeat/tests/system/test_reload_inputs.py @@ -1,4 +1,5 @@ import os +import pytest import time from filebeat import BaseTest @@ -249,6 +250,8 @@ def test_load_configs(self): assert output[0]["message"] == first_line assert output[1]["message"] == second_line + # 1/20 build fails https://github.com/elastic/beats/issues/21307 + @pytest.mark.flaky(reruns=1, reruns_delay=10) def test_reload_same_config(self): """ Test reload same config with same file but different config. Makes sure reloading also works on conflicts. diff --git a/x-pack/auditbeat/tests/system/test_metricsets.py b/x-pack/auditbeat/tests/system/test_metricsets.py index 26bddcef785..d60c1b82ef8 100644 --- a/x-pack/auditbeat/tests/system/test_metricsets.py +++ b/x-pack/auditbeat/tests/system/test_metricsets.py @@ -1,6 +1,7 @@ import jinja2 import os import platform +import pytest import sys import time import unittest @@ -42,6 +43,8 @@ def test_metricset_login(self): # Metricset is beta and that generates a warning, TODO: remove later self.check_metricset("system", "login", COMMON_FIELDS + fields, config, warnings_allowed=True) + # 1/20 build fails https://github.com/elastic/beats/issues/21308 + @pytest.mark.flaky(reruns=1, reruns_delay=10) @unittest.skipIf(sys.platform == "win32", "Not implemented for Windows") @unittest.skipIf(sys.platform.startswith('linux') and not (os.path.isdir("/var/lib/dpkg") or os.path.isdir("/var/lib/rpm")), "Only implemented for dpkg and rpm") From cdfa559a7e230d4162258b639131e589a2336614 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Wed, 30 Sep 2020 15:15:17 +0200 Subject: [PATCH 15/17] docs: Prepare Changelog for 7.9.2 (#21229) (#21403) * Close changelog for 7.9.2 * Update CHANGELOG.next.asciidoc * Apply suggestions from code review --- CHANGELOG.asciidoc | 41 +++++++++++++++++++++++++++++++++++++++++ CHANGELOG.next.asciidoc | 4 ++++ 2 files changed, 45 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index b0b31734005..c4d0f48005f 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -3,6 +3,47 @@ :issue: https://github.com/elastic/beats/issues/ :pull: https://github.com/elastic/beats/pull/ +[[release-notes-7.9.2]] +=== Beats version 7.9.2 +https://github.com/elastic/beats/compare/v7.9.1...v7.9.2[View commits] + +==== Breaking changes + +*Affecting all Beats* + +- Autodiscover doesn't generate any configuration when a variable is missing. Previously it generated an incomplete configuration. {pull}20898[20898] + +==== Bugfixes + +*Affecting all Beats* + +- Explicitly detect missing variables in autodiscover configuration, log them at the debug level. {issue}20568[20568] {pull}20898[20898] +- Fix `libbeat.output.write.bytes` and `libbeat.output.read.bytes` metrics of the Elasticsearch output. {issue}20752[20752] {pull}21197[21197] + +*Filebeat* + +- Provide backwards compatibility for the `set` processor when Elasticsearch is less than 7.9.0. {pull}20908[20908] +- Fix an error updating file size being logged when EOF is reached. {pull}21048[21048] +- Fix error when processing AWS Cloudtrail Digest logs. {pull}21086[21086] {issue}20943[20943] + +*Metricbeat* + +- The Kibana collector applies backoff when errored at getting usage stats {pull}20772[20772] +- The `elasticsearch/index` metricset only requests wildcard expansion for hidden indices if the monitored Elasticsearch cluster supports it. {pull}20938[20938] +- Fix panic index out of range error when getting AWS account name. {pull}21101[21101] {issue}21095[21095] +- Handle missing counters in the application_pool metricset. {pull}21071[21071] + +*Functionbeat* + +- Do not need Google credentials if not required for the operation. {issue}17329[17329] {pull}21072[21072] +- Fix dependency issues of GCP functions. {issue}20830[20830] {pull}21070[21070] + +==== Added + +*Affecting all Beats* + +- Add container ECS fields in kubernetes metadata. {pull}20984[20984] + [[release-notes-7.9.1]] === Beats version 7.9.1 https://github.com/elastic/beats/compare/v7.9.0...v7.9.1[View commits] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1b5b1bd16bb..223dace9e49 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -23,6 +23,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Autodiscover doesn't generate any configuration when a variable is missing. Previously it generated an incomplete configuration. {pull}20898[20898] - Remove redundant `cloudfoundry.*.timestamp` fields. This value is set in `@timestamp`. {pull}21175[21175] - Allow embedding of CAs, Certificate of private keys for anything that support TLS in ouputs and inputs https://github.com/elastic/beats/pull/21179 +- Update to Golang 1.12.1. {pull}11330[11330] +- Disable Alibaba Cloud and Tencent Cloud metadata providers by default. {pull}13812[12812] *Auditbeat* @@ -74,6 +76,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix PANW field spelling "veredict" to "verdict" on event.action {pull}18808[18808] - Removed experimental modules `citrix`, `kaspersky`, `rapid7` and `tenable`. {pull}20706[20706] - Add support for GMT timezone offsets in `decode_cef`. {pull}20993[20993] +- Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547] *Heartbeat* @@ -356,6 +359,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix timestamp handling in remote_write. {pull}21166[21166] - Fix remote_write flaky test. {pull}21173[21173] - Visualization title fixes in aws, azure and googlecloud compute dashboards. {pull}21098[21098] +- Add a switch to the driver definition on SQL module to use pretty names {pull}17378[17378] *Packetbeat* From cba5245b313e40d409b14b362859b427841df0b3 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Wed, 30 Sep 2020 16:21:34 +0200 Subject: [PATCH 16/17] docs: add link to release notes for 7.9.2 (#21405) (#21419) --- libbeat/docs/release.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc index 4215186d430..24e0ee43651 100644 --- a/libbeat/docs/release.asciidoc +++ b/libbeat/docs/release.asciidoc @@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read <> for more detail about changes that affect upgrade. +* <> * <> * <> * <> From cadd5ff70e679f76e41f32daa1d56734ddf3ae8c Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 30 Sep 2020 16:49:44 +0100 Subject: [PATCH 17/17] [JJBB] Set shallow cloning to 10 (#21409) --- .ci/jobs/apm-beats-update.yml | 2 +- .ci/jobs/beats-tester.yml | 2 +- .ci/jobs/beats-windows-mbp.yml | 2 +- .ci/jobs/beats.yml | 2 +- .ci/jobs/golang-crossbuild-mbp.yml | 2 +- .ci/jobs/packaging.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.ci/jobs/apm-beats-update.yml b/.ci/jobs/apm-beats-update.yml index 8bdc322f65a..2ae688ffab7 100644 --- a/.ci/jobs/apm-beats-update.yml +++ b/.ci/jobs/apm-beats-update.yml @@ -48,7 +48,7 @@ before: true prune: true shallow-clone: true - depth: 3 + depth: 10 do-not-fetch-tags: true submodule: disable: false diff --git a/.ci/jobs/beats-tester.yml b/.ci/jobs/beats-tester.yml index 522abfa5e5c..808123a225e 100644 --- a/.ci/jobs/beats-tester.yml +++ b/.ci/jobs/beats-tester.yml @@ -44,7 +44,7 @@ before: true prune: true shallow-clone: true - depth: 3 + depth: 10 do-not-fetch-tags: true submodule: disable: false diff --git a/.ci/jobs/beats-windows-mbp.yml b/.ci/jobs/beats-windows-mbp.yml index 64efa009979..7ea26c9ba18 100644 --- a/.ci/jobs/beats-windows-mbp.yml +++ b/.ci/jobs/beats-windows-mbp.yml @@ -44,7 +44,7 @@ before: true prune: true shallow-clone: true - depth: 4 + depth: 10 do-not-fetch-tags: true submodule: disable: false diff --git a/.ci/jobs/beats.yml b/.ci/jobs/beats.yml index 1e393bab6b9..3e4c8e50980 100644 --- a/.ci/jobs/beats.yml +++ b/.ci/jobs/beats.yml @@ -46,7 +46,7 @@ before: true prune: true shallow-clone: true - depth: 3 + depth: 10 do-not-fetch-tags: true submodule: disable: false diff --git a/.ci/jobs/golang-crossbuild-mbp.yml b/.ci/jobs/golang-crossbuild-mbp.yml index 46303790610..45175d169f6 100644 --- a/.ci/jobs/golang-crossbuild-mbp.yml +++ b/.ci/jobs/golang-crossbuild-mbp.yml @@ -31,7 +31,7 @@ before: true prune: true shallow-clone: true - depth: 4 + depth: 10 do-not-fetch-tags: true submodule: disable: false diff --git a/.ci/jobs/packaging.yml b/.ci/jobs/packaging.yml index 0dce4d4672b..fd6fb9f90c6 100644 --- a/.ci/jobs/packaging.yml +++ b/.ci/jobs/packaging.yml @@ -44,7 +44,7 @@ before: true prune: true shallow-clone: true - depth: 3 + depth: 10 do-not-fetch-tags: true submodule: disable: false