Merge branch 'master' into feature/indexing_permissions

.go-version

@@ -1 +1 @@
-1.15.8
+1.16.2

.mergify.yml

@@ -29,3 +29,23 @@ pull_request_rules:
  backport:
  branches:
  - "7.12"
+ - name: automatic merge for 7.x when CI passes
+ conditions:
+ - check-success=fleet-server/pr-merge
+ - check-success=CLA
+ - base=7.x
+ actions:
+ merge:
+ method: squash
+ strict: smart+fasttrack
+ priority: low
+ - name: automatic merge for 7.12 when CI passes
+ conditions:
+ - check-success=fleet-server/pr-merge
+ - check-success=CLA
+ - base=7.12
+ actions:
+ merge:
+ method: squash
+ strict: smart+fasttrack
+ priority: low

Makefile

@@ -3,11 +3,13 @@ DEFAULT_VERSION=$(shell awk '/const defaultVersion/{print $$NF}' main.go | tr -d
 TARGET_ARCH_386=x86
 TARGET_ARCH_amd64=x86_64
 TARGET_ARCH_arm64=arm64
-PLATFORMS ?= darwin/amd64 linux/386 linux/amd64 linux/arm64 windows/386 windows/amd64
+PLATFORMS ?= darwin/amd64 darwin/arm64 linux/386 linux/amd64 linux/arm64 windows/386 windows/amd64
 BUILDMODE_linux_amd64=-buildmode=pie
 BUILDMODE_linux_arm64=-buildmode=pie
 BUILDMODE_windows_386=-buildmode=pie
 BUILDMODE_windows_amd64=-buildmode=pie
+BUILDMODE_darwin_amd64=-buildmode=pie
+BUILDMODE_darwin_arm64=-buildmode=pie
 
 ifeq ($(SNAPSHOT),true)
 VERSION=${DEFAULT_VERSION}-SNAPSHOT

NOTICE.txt

@@ -11,11 +11,11 @@ Third party libraries used by the Elastic Beats project:
 
 --------------------------------------------------------------------------------
 Dependency : github.com/aleksmaus/generate
-Version: v0.0.0-20201213151810-c5bc68a6a42f
+Version: v0.0.0-20210326194607-c630e07a2742
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/aleksmaus/generate@v0.0.0-20201213151810-c5bc68a6a42f/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/aleksmaus/generate@v0.0.0-20210326194607-c630e07a2742/LICENSE.txt:
 
 MIT License
 

README.md

@@ -21,13 +21,7 @@ yarn kbn bootstrap
 
 This will take a while the first time it is run. An error might be return in case not a valid node version is installed. Use nvm to install the correct version.
 
-Next, Kibana and Elasticsearch must be started. On the Kibana side, an additional configuration flag is required. I personally put this into `config/kibana.dev.yml` in the Kibana repo as this file is ignored from git. The content to put in there is:
-
-```
-xpack.fleet.agents.fleetServerEnabled: true
-```
-
-This enables the fleet-server setup in Kibana. Now the following two commands must be run in parallel:
+Now the following two commands must be run in parallel:
 
 ```
 # Start ES
@@ -39,6 +33,8 @@ yarn start --no-base-path
 
 As soon as all is running, go to `http://localhost:5601`, enter `elastic/changeme` as credential and navigate to Fleet. Trigger the Fleet setup. As soon as this is completed, copy the `policy id` and `enrollment token` for the fleet-server policy. The policy id can be copied from the URL, the enrollment token can be found in the Enrollment Token list. 
 
+NOTE: This step can be skipped if the full command below for the Elastic Agent is used.
+
 Now Kibana is running and ready. The next step is to setup Elastic Agent.
 
 ## Beats repo
@@ -49,20 +45,38 @@ To build the Elastic Agent from source, check out the beats repository. Navigate
 SNAPSHOT=true DEV=true PLATFORMS=darwin mage package
 ```
 
-The above assumes you are running on OS X. Put the platform in you are running on. This speeds up packaging as it only builds it for your platform. As soon as this is completed (it might take a while for the first time) navigate to `build/distributions` and unpackage the `.tar.gz`. Navigate into the elastic-agent directory and start the Elastic Agent:
+The above assumes you are running on OS X. Put the platform in you are running on. This speeds up packaging as it only builds it for your platform. As soon as this is completed (it might take a while for the first time) navigate to `build/distributions` and unpackage the `.tar.gz`. Change working directory to the elastic-agent directory and start the Elastic Agent:
 
 ```
-sudo ./elastic-agent -v
+KIBANA_HOST=http://localhost:5601 KIBANA_USERNAME=elastic KIBANA_PASSWORD=changeme ELASTICSEARCH_HOST=http://localhost:9200 ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme KIBANA_FLEET_SETUP=1 FLEET_SERVER_ENABLE=1 sudo ./elastic-agent container
 ```
 
-Currently a second command has to be run in parallel to setup fleet-server. Take the enrollment token and policy id you copied from Kibana and replace it in the command below:
+This will start up Elastic Agent with fleet-server and directly enroll it. In addition Fleet is setup inside of Kibana.
+
+## Running Elastic Agent with fleet-server in container
+
+If you want to run Elastic Agent and fleet-server in a container but built Kibana from source, you have to add the following to your `config/kibana.dev.yml`:
 
 ```
-sudo ./elastic-agent enroll --enrollment-token {enrollment-token} --fleet-server http://elastic:changeme@localhost:9200 --fleet-server-policy {fleet-server-policy-id}
+server.host: 0.0.0.0
 ```
 
-This will start up fleet-server and the command should complete. After this, navigate to Kibana and check if the Elastic Agent with fleet-server shows up.
+This makes sure, Kibana is accessible from the container. Start Kibana as before but for Elasticsearch, run the following command:
+
+```
+yarn es snapshot -E xpack.security.authc.api_key.enabled=true -E http.host=0.0.0.0
+```
+
+This makes sure also Elasticsearch is accessible to the container.
+
+Start the Elastic Agent with the following command:
+
+```
+docker run -e KIBANA_HOST=http://{YOUR-IP}:5601 -e KIBANA_USERNAME=elastic -e KIBANA_PASSWORD=changeme -e ELASTICSEARCH_HOST=http://{YOUR-IP}:9200 -e ELASTICSEARCH_USERNAME=elastic -e ELASTICSEARCH_PASSWORD=changeme -e KIBANA_FLEET_SETUP=1 -e FLEET_SERVER_ENABLE=1 -e FLEET_SERVER_INSECURE_HTTP=1 docker.elastic.co/beats/elastic-agent:8.0.0-SNAPSHOT
+```
 
+Replace {YOUR-IP} with the IP address of your machine.
 
+## fleet-server repo
 
-sudo ./elastic-agent -v
+By default the above will download the most recent snapshot build for fleet-server. To use your own development build, run `make release` in the fleet-server repository, go to `build/distributions` and copy the `.tar.gz` and `sha512` file to the `data/elastic-agent-{hash}/downloads` inside the elastic-agent directory. Now you run with your own build of fleet-server.

cmd/fleet/bulkCheckin.go

@@ -12,6 +12,7 @@ import (
 
  "github.com/elastic/fleet-server/v7/internal/pkg/bulk"
  "github.com/elastic/fleet-server/v7/internal/pkg/dl"
+ "github.com/elastic/fleet-server/v7/internal/pkg/sqn"
 
  "github.com/rs/zerolog/log"
 )
@@ -22,7 +23,7 @@ const kBulkCheckinFlushInterval = 10 * time.Second
 
 type PendingData struct {
  fields Fields
- seqNo int64
+ seqNo sqn.SeqNo
 }
 
 type BulkCheckin struct {
@@ -38,7 +39,7 @@ func NewBulkCheckin(bulker bulk.Bulk) *BulkCheckin {
  }
 }
 
-func (bc *BulkCheckin) CheckIn(id string, fields Fields, seqno int64) error {
+func (bc *BulkCheckin) CheckIn(id string, fields Fields, seqno sqn.SeqNo) error {
 
  if fields == nil {
  fields = make(Fields)
@@ -93,7 +94,7 @@ func (bc *BulkCheckin) flush(ctx context.Context) error {
  for id, pendingData := range pending {
  doc := pendingData.fields
  doc[dl.FieldUpdatedAt] = time.Now().UTC().Format(time.RFC3339)
- if pendingData.seqNo >= 0 {
+ if pendingData.seqNo.IsSet() {
  doc[dl.FieldActionSeqNo] = pendingData.seqNo
  }
 

cmd/fleet/handleCheckin.go

@@ -22,6 +22,7 @@ import (
  "github.com/elastic/fleet-server/v7/internal/pkg/monitor"
  "github.com/elastic/fleet-server/v7/internal/pkg/policy"
  "github.com/elastic/fleet-server/v7/internal/pkg/smap"
+ "github.com/elastic/fleet-server/v7/internal/pkg/sqn"
 
  "github.com/julienschmidt/httprouter"
  "github.com/rs/zerolog/log"
@@ -201,7 +202,7 @@ func (ct *CheckinT) _handleCheckin(w http.ResponseWriter, r *http.Request, id st
 }
 
 // Resolve AckToken from request, fallback on the agent record
-func (ct *CheckinT) resolveSeqNo(ctx context.Context, req CheckinRequest, agent *model.Agent) (seqno int64, err error) {
+func (ct *CheckinT) resolveSeqNo(ctx context.Context, req CheckinRequest, agent *model.Agent) (seqno sqn.SeqNo, err error) {
  // Resolve AckToken from request, fallback on the agent record
  ackToken := req.AckToken
  seqno = agent.ActionSeqNo
@@ -217,16 +218,16 @@ func (ct *CheckinT) resolveSeqNo(ctx context.Context, req CheckinRequest, agent
  return
  }
  }
- seqno = sn
+ seqno = []int64{sn}
  }
  return seqno, nil
 }
 
-func (ct *CheckinT) fetchAgentPendingActions(ctx context.Context, seqno int64, agentId string) ([]model.Action, error) {
+func (ct *CheckinT) fetchAgentPendingActions(ctx context.Context, seqno sqn.SeqNo, agentId string) ([]model.Action, error) {
  now := time.Now().UTC().Format(time.RFC3339)
 
  return dl.FindActions(ctx, ct.bulker, dl.QueryAgentActions, map[string]interface{}{
- dl.FieldSeqNo: seqno,
+ dl.FieldSeqNo: seqno.Get(0),
  dl.FieldMaxSeqNo: ct.gcp.GetCheckpoint(),
  dl.FieldExpiration: now,
  dl.FieldAgents: []string{agentId},

cmd/fleet/handleEnroll.go

@@ -19,6 +19,7 @@ import (
  "github.com/elastic/fleet-server/v7/internal/pkg/config"
  "github.com/elastic/fleet-server/v7/internal/pkg/dl"
  "github.com/elastic/fleet-server/v7/internal/pkg/model"
+ "github.com/elastic/fleet-server/v7/internal/pkg/sqn"
 
  "github.com/elastic/go-elasticsearch/v8"
  "github.com/gofrs/uuid"
@@ -192,7 +193,7 @@ func _enroll(ctx context.Context, bulker bulk.Bulk, c cache.Cache, req EnrollReq
  EnrolledAt: now.UTC().Format(time.RFC3339),
  LocalMetadata: localMeta,
  AccessApiKeyId: accessApiKey.Id,
- ActionSeqNo: dl.UndefinedSeqNo,
+ ActionSeqNo: []int64{sqn.UndefinedSeqNo},
  }
 
  err = createFleetAgent(ctx, bulker, agentId, agentData)

cmd/fleet/main.go

@@ -50,13 +50,13 @@ func installSignalHandler() context.Context {
 func makeCache(cfg *config.Config) (cache.Cache, error) {
 
  log.Info().
- Int64("numCounters", cfg.Cache.NumCounters).
- Int64("maxCost", cfg.Cache.MaxCost).
+ Int64("numCounters", cfg.Inputs[0].Cache.NumCounters).
+ Int64("maxCost", cfg.Inputs[0].Cache.MaxCost).
  Msg("makeCache")
 
  cacheCfg := cache.Config{
- NumCounters: cfg.Cache.NumCounters,
- MaxCost: cfg.Cache.MaxCost,
+ NumCounters: cfg.Inputs[0].Cache.NumCounters,
+ MaxCost: cfg.Inputs[0].Cache.MaxCost,
  }
 
  return cache.New(cacheCfg)
@@ -422,11 +422,14 @@ func (f *FleetServer) Run(ctx context.Context) error {
  }
 
  // Restart profiler
- if curCfg == nil || curCfg.Inputs[0].Server.Profile.Bind != newCfg.Inputs[0].Server.Profile.Bind {
+ if curCfg == nil || curCfg.Inputs[0].Server.Profile.Enabled != newCfg.Inputs[0].Server.Profile.Enabled || curCfg.Inputs[0].Server.Profile.Bind != newCfg.Inputs[0].Server.Profile.Bind {
  stop(proCancel, proEg)
- proEg, proCancel = start(ctx, func(ctx context.Context) error {
- return profile.RunProfiler(ctx, newCfg.Inputs[0].Server.Profile.Bind)
- }, ech)
+ proEg, proCancel = nil, nil
+ if newCfg.Inputs[0].Server.Profile.Enabled {
+ proEg, proCancel = start(ctx, func(ctx context.Context) error {
+ return profile.RunProfiler(ctx, newCfg.Inputs[0].Server.Profile.Bind)
+ }, ech)
+ }
  }
 
  // Restart server
@@ -493,7 +496,7 @@ func (f *FleetServer) runServer(ctx context.Context, cfg *config.Config) (err er
  g, ctx := errgroup.WithContext(ctx)
 
  // Coordinator policy monitor
- pim, err := monitor.New(dl.FleetPolicies, es)
+ pim, err := monitor.New(dl.FleetPolicies, es, monitor.WithFetchSize(cfg.Inputs[0].Monitor.FetchSize))
  if err != nil {
  return err
  }
@@ -515,8 +518,7 @@ func (f *FleetServer) runServer(ctx context.Context, cfg *config.Config) (err er
  var ad *action.Dispatcher
  var tr *action.TokenResolver
 
- // Behind the feature flag
- am, err = monitor.NewSimple(dl.FleetActions, es, monitor.WithExpiration(true))
+ am, err = monitor.NewSimple(dl.FleetActions, es, monitor.WithExpiration(true), monitor.WithFetchSize(cfg.Inputs[0].Monitor.FetchSize))
  if err != nil {
  return err
  }

fleet-server.yml

@@ -12,9 +12,13 @@ fleet:
 
 # Input config provided by the Elastic Agent for the server
 #inputs:
-# - type:
-# policy:
+# - type: fleet-server
 # server:
+# host: localhost
+# port: 8220
+# cache:
+# num_counters: 500000 # 10x times expected count
+# max_cost: 50 * 1024 * 1024 # 50MiB cache size
 
 logging:
  to_stderr: true

go.mod

@@ -3,7 +3,7 @@ module github.com/elastic/fleet-server/v7
 go 1.15
 
 require (
- github.com/aleksmaus/generate v0.0.0-20201213151810-c5bc68a6a42f
+ github.com/aleksmaus/generate v0.0.0-20210326194607-c630e07a2742
  github.com/dgraph-io/ristretto v0.0.3
  github.com/elastic/beats/v7 v7.11.1
  github.com/elastic/elastic-agent-client/v7 v7.0.0-20200709172729-d43b7ad5833a

go.sum

@@ -106,8 +106,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/aleksmaus/generate v0.0.0-20201213151810-c5bc68a6a42f h1:wr9LrxkE1Ai416C/mis1gEDsXrbERHGufCmf7xuYwI4=
-github.com/aleksmaus/generate v0.0.0-20201213151810-c5bc68a6a42f/go.mod h1:lvlu2Ij1bLmxB8RUWyw5IQ4/JcLX60eYhLiBmvImnhk=
+github.com/aleksmaus/generate v0.0.0-20210326194607-c630e07a2742 h1:lDBhj+4eBCS9tNiJLXrNbvwO5xwkn2/kjvy+tO+PWlI=
+github.com/aleksmaus/generate v0.0.0-20210326194607-c630e07a2742/go.mod h1:lvlu2Ij1bLmxB8RUWyw5IQ4/JcLX60eYhLiBmvImnhk=
 github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 h1:7rj9qZ63knnVo2ZeepYHvHuRdG76f3tRUTdIQDzRBeI=
 github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20/go.mod h1:cI59GRkC2FRaFYtgbYEqMlgnnfvAwXzjojyZKXwklNg=
 github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43 h1:WFwa9pqou0Nb4DdfBOyaBTH0GqLE74Qwdf61E7ITHwQ=

internal/pkg/action/dispatcher.go

@@ -11,13 +11,14 @@ import (
  "github.com/elastic/fleet-server/v7/internal/pkg/es"
  "github.com/elastic/fleet-server/v7/internal/pkg/model"
  "github.com/elastic/fleet-server/v7/internal/pkg/monitor"
+ "github.com/elastic/fleet-server/v7/internal/pkg/sqn"
 
  "github.com/rs/zerolog/log"
 )
 
 type Sub struct {
  agentId string
- seqNo int64
+ seqNo sqn.SeqNo
  ch chan []model.Action
 }
 
@@ -50,7 +51,7 @@ func (d *Dispatcher) Run(ctx context.Context) (err error) {
  }
 }
 
-func (d *Dispatcher) Subscribe(agentId string, seqNo int64) *Sub {
+func (d *Dispatcher) Subscribe(agentId string, seqNo sqn.SeqNo) *Sub {
  cbCh := make(chan []model.Action, 1)
 
  sub := Sub{

internal/pkg/config/config.go

@@ -27,15 +27,13 @@ type Config struct {
  Inputs []Input `config:"inputs"`
  Logging Logging `config:"logging"`
  HTTP HTTP `config:"http"`
- Cache Cache `config:"cache"`
 }
 
 // InitDefaults initializes the defaults for the configuration.
 func (c *Config) InitDefaults() {
  c.Inputs = make([]Input, 1)
  c.Inputs[0].InitDefaults()
  c.HTTP.InitDefaults()
- c.Cache.InitDefaults()
 }
 
 // Validate ensures that the configuration is valid.