From 129eefedce39c7c6c43b57c0588e0df41c4d5e09 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Wed, 1 Jun 2022 01:32:58 -0400 Subject: [PATCH 01/89] [Automation] Update elastic stack version to 8.3.0-4644f7c7 for testing (#1513) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index 087dd342c..0ab115fc9 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-a2430aec-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-4644f7c7-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From 0d99f4b37a8053b0a8bd87c563c23d64282feadd Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Thu, 2 Jun 2022 01:33:41 -0400 Subject: [PATCH 02/89] [Automation] Update elastic stack version to 8.3.0-17c7e5c0 for testing (#1517) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index 0ab115fc9..26aa0cca4 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-4644f7c7-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-17c7e5c0-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From 03a91343a89032be5859a871229a829e3ec8ae92 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Fri, 3 Jun 2022 01:32:09 -0400 Subject: [PATCH 03/89] [Automation] Update elastic stack version to 8.3.0-378ebcca for testing (#1520) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index 26aa0cca4..53b2f7267 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-17c7e5c0-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-378ebcca-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From 7fcb743d4db21b36c2dedaebb408b191ed53e038 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Mon, 6 Jun 2022 01:34:10 -0400 Subject: [PATCH 04/89] [Automation] Update elastic stack version to 8.3.0-9596ee28 for testing (#1531) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index 53b2f7267..5be6abefe 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-378ebcca-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-9596ee28-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From 7192db93ca0bba46ac2025432870fe27aad36ed8 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Tue, 7 Jun 2022 01:32:28 -0400 Subject: [PATCH 05/89] [Automation] Update elastic stack version to 8.3.0-f530a93f for testing (#1534) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index 5be6abefe..0fa3cb9b9 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-9596ee28-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-f530a93f-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From 96976f82d1c7ae26b77e389e9c8c7cdea80a1bd1 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Wed, 8 Jun 2022 01:31:08 -0400 Subject: [PATCH 06/89] [Automation] Update elastic stack version to 8.3.0-f13a89b4 for testing (#1537) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index 0fa3cb9b9..cd6df38bc 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-f530a93f-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-f13a89b4-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From 26f08df24bf9f75b7ce13c370699b2b7887f434a Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Thu, 9 Jun 2022 01:33:19 -0400 Subject: [PATCH 07/89] [Automation] Update elastic stack version to 8.3.0-3c22a7f7 for testing (#1543) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index cd6df38bc..85475d155 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-f13a89b4-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-3c22a7f7-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From b9f7464bf360cdda1708688262db308b73ffeb37 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Fri, 10 Jun 2022 01:33:51 -0400 Subject: [PATCH 08/89] [Automation] Update elastic stack version to 8.3.0-158b3cba for testing (#1545) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index 85475d155..7463f9fb1 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-3c22a7f7-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-158b3cba-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From 7d372c60de088c7b2b05190f67e11be963bede5c Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Mon, 13 Jun 2022 01:32:17 -0400 Subject: [PATCH 09/89] [Automation] Update elastic stack version to 8.3.0-db012561 for testing (#1551) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index 7463f9fb1..4d8e4fb1f 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-158b3cba-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-db012561-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From 37d116d27475a7f88363dc682b59815295f29c32 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Tue, 14 Jun 2022 01:30:52 -0400 Subject: [PATCH 10/89] [Automation] Update elastic stack version to 8.3.0-e4e76686 for testing (#1554) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index 4d8e4fb1f..63bc0efe8 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-db012561-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-e4e76686-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From 6c802537e68116d72c99b5e57a10d7c447b5f214 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Wed, 15 Jun 2022 01:32:40 -0400 Subject: [PATCH 11/89] [Automation] Update elastic stack version to 8.3.0-460dc667 for testing (#1557) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index 63bc0efe8..a0977dabd 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-e4e76686-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-460dc667-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From 91518f39c5dfd0cd87c95f068e03579362015634 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Thu, 16 Jun 2022 01:31:05 -0400 Subject: [PATCH 12/89] [Automation] Update elastic stack version to 8.3.0-fe62331f for testing (#1561) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index a0977dabd..cba60ccf0 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-460dc667-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-fe62331f-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From 282fb101d3763980c0fe7cba43ab21d72d19eb6c Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Mon, 20 Jun 2022 01:32:44 -0400 Subject: [PATCH 13/89] [Automation] Update elastic stack version to 8.3.0-75d7500e for testing (#1570) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index cba60ccf0..a15594501 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-fe62331f-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-75d7500e-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From a322ddc3e67afcb26ee261b544427fa7bf845174 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Tue, 21 Jun 2022 01:36:10 -0400 Subject: [PATCH 14/89] [Automation] Update elastic stack version to 8.3.0-6d4bb852 for testing (#1573) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index a15594501..2d6840113 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-75d7500e-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-6d4bb852-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From 5fa764e1c0d5a8f7b8392206fe1a396546f7e7fc Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Tue, 21 Jun 2022 14:05:25 -0400 Subject: [PATCH 15/89] [Automation] Update elastic stack version to 8.3.0-e0057e17 for testing (#1576) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index 2d6840113..1cef2a127 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-6d4bb852-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-e0057e17-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From ce387fa974ab7478dc19a6c825771638cd22cd79 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Thu, 23 Jun 2022 01:35:26 -0400 Subject: [PATCH 16/89] [Automation] Update elastic stack version to 8.3.0-47d97929 for testing (#1584) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index 1cef2a127..7ce642f39 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-e0057e17-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-47d97929-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From d14e71ef2464802a2ad88ea7be5449451d000d7b Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 23 Jun 2022 11:33:52 +0100 Subject: [PATCH 17/89] ci: enable build notifications as GitHub issues (#1582) (#1587) (cherry picked from commit a51ed0ff50333763e8171a25a4562531f8c84944) Co-authored-by: Victor Martinez --- .ci/Jenkinsfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 787654fb5..125dd2073 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -133,7 +133,9 @@ pipeline { } post { cleanup { - notifyBuildResult(prComment: true) + notifyBuildResult(prComment: true, + githubIssue: isBranch() && currentBuild.currentResult != "SUCCESS", + githubLabels: 'Team:Elastic-Agent-Control-Plane') } } } From 211e9b93e3cd91f2472ba0ac590be60138d16209 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Fri, 24 Jun 2022 01:31:26 -0400 Subject: [PATCH 18/89] [Automation] Update elastic stack version to 8.3.0-3b390c7b for testing (#1590) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index 7ce642f39..bf2c29120 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-47d97929-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-3b390c7b-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From d7928b4781cf2d7972811fcbca07ce59bd0f77ca Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Mon, 27 Jun 2022 01:32:59 -0400 Subject: [PATCH 19/89] [Automation] Update elastic stack version to 8.3.0-fdf97a2e for testing (#1600) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index bf2c29120..a6f1545e1 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-3b390c7b-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-fdf97a2e-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From 88dd2b6f06e66031957233d7e0c81e1366ddc480 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 15:17:25 +0000 Subject: [PATCH 20/89] Bugfix: Avoid panic when check-in setup duration is longer than poll duration (#1604) (cherry picked from commit 39588201fc842bd33c1b731d67b91d7707566bf9) Co-authored-by: Sean Cunningham --- internal/pkg/api/handleCheckin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/api/handleCheckin.go b/internal/pkg/api/handleCheckin.go index 26eb84a8e..07f89335c 100644 --- a/internal/pkg/api/handleCheckin.go +++ b/internal/pkg/api/handleCheckin.go @@ -566,7 +566,7 @@ func calcPollDuration(zlog zerolog.Logger, cfg *config.Server, setupDuration tim if setupDuration >= pollDuration { // We took so long to setup that we need to exit immediately - pollDuration = 0 + pollDuration = time.Millisecond zlog.Warn(). Dur("setupDuration", setupDuration). Dur("pollDuration", cfg.Timeouts.CheckinLongPoll). From 6059b03d9e6bbaf9ec72df49505e5b70e38daa2e Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 20:33:57 +0100 Subject: [PATCH 21/89] ci: enable flaky test detector (#1589) (#1602) (cherry picked from commit bdd20a1d5d74444c74a6b4d3c5fa34fedbca5997) Co-authored-by: Victor Martinez Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .ci/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index 125dd2073..359f32833 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -134,6 +134,7 @@ pipeline { post { cleanup { notifyBuildResult(prComment: true, + analyzeFlakey: !isTag(), jobName: getFlakyJobName(withBranch: (isPR() ? env.CHANGE_TARGET : env.BRANCH_NAME)), githubIssue: isBranch() && currentBuild.currentResult != "SUCCESS", githubLabels: 'Team:Elastic-Agent-Control-Plane') } From cddd9d7773497b0c97825fee10753d016434ce73 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Tue, 28 Jun 2022 14:58:22 -0400 Subject: [PATCH 22/89] Update to 8.3.1 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index 1bebbb425..11d4401ba 100644 --- a/version/version.go +++ b/version/version.go @@ -4,4 +4,4 @@ package version -const DefaultVersion = "8.3.0" +const DefaultVersion = "8.3.1" From 152c1d945a21f8561c7ce47eadba90c784b43384 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Tue, 28 Jun 2022 15:00:18 -0400 Subject: [PATCH 23/89] Udpate doc --- version/version.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/version/version.go b/version/version.go index 11d4401ba..f8bf9d8a0 100644 --- a/version/version.go +++ b/version/version.go @@ -4,4 +4,6 @@ package version +// DefaultVersion is the current release version of Fleet-server, this version must match the +// Elastic Agent version. const DefaultVersion = "8.3.1" From a487054615863a0582fffe331d9552b631349336 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Wed, 29 Jun 2022 01:31:50 -0400 Subject: [PATCH 24/89] [Automation] Update elastic stack version to 8.3.0-729c5564 for testing (#1615) Co-authored-by: apmmachine --- dev-tools/integration/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/integration/.env b/dev-tools/integration/.env index a6f1545e1..54f9fdd11 100644 --- a/dev-tools/integration/.env +++ b/dev-tools/integration/.env @@ -1,4 +1,4 @@ -ELASTICSEARCH_VERSION=8.3.0-fdf97a2e-SNAPSHOT +ELASTICSEARCH_VERSION=8.3.0-729c5564-SNAPSHOT ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=changeme TEST_ELASTICSEARCH_HOSTS=localhost:9200 \ No newline at end of file From 1bbad27bcec3667081c6994fbc18bce7dd6add52 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 14 Jul 2022 07:19:28 +0200 Subject: [PATCH 25/89] debug api key creation --- internal/pkg/api/handleCheckin.go | 1 - internal/pkg/api/handleEnroll.go | 1 - internal/pkg/apikey/create.go | 1 - internal/pkg/apikey/invalidate.go | 1 - internal/pkg/policy/policy_output.go | 23 +++++++++++++++++------ 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/internal/pkg/api/handleCheckin.go b/internal/pkg/api/handleCheckin.go index e28fdeac1..7e73f18e1 100644 --- a/internal/pkg/api/handleCheckin.go +++ b/internal/pkg/api/handleCheckin.go @@ -458,7 +458,6 @@ func processPolicy(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, a // Iterate through the policy outputs and prepare them for _, policyOutput := range pp.Outputs { err = policyOutput.Prepare(ctx, zlog, bulker, &agent, outputs) - if err != nil { return nil, err } diff --git a/internal/pkg/api/handleEnroll.go b/internal/pkg/api/handleEnroll.go index a3c2f9833..d97943b53 100644 --- a/internal/pkg/api/handleEnroll.go +++ b/internal/pkg/api/handleEnroll.go @@ -53,7 +53,6 @@ type EnrollerT struct { } func NewEnrollerT(verCon version.Constraints, cfg *config.Server, bulker bulk.Bulk, c cache.Cache) (*EnrollerT, error) { - log.Info(). Interface("limits", cfg.Limits.EnrollLimit). Msg("Setting config enroll_limit") diff --git a/internal/pkg/apikey/create.go b/internal/pkg/apikey/create.go index f3cee99f8..de61390c3 100644 --- a/internal/pkg/apikey/create.go +++ b/internal/pkg/apikey/create.go @@ -42,7 +42,6 @@ func Create(ctx context.Context, client *elasticsearch.Client, name, ttl, refres bytes.NewReader(body), opts..., ) - if err != nil { return nil, err } diff --git a/internal/pkg/apikey/invalidate.go b/internal/pkg/apikey/invalidate.go index 421662388..6c5d5d304 100644 --- a/internal/pkg/apikey/invalidate.go +++ b/internal/pkg/apikey/invalidate.go @@ -38,7 +38,6 @@ func Invalidate(ctx context.Context, client *elasticsearch.Client, ids ...string bytes.NewReader(body), opts..., ) - if err != nil { return fmt.Errorf("InvalidateAPIKey: %w", err) } diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index 8115d22ec..e55ae9441 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -41,7 +41,11 @@ type PolicyOutput struct { func (p *PolicyOutput) Prepare(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, agent *model.Agent, outputMap smap.Map) error { switch p.Type { case OutputTypeElasticsearch: - zlog.Debug().Msg("preparing elasticsearch output") + zlog = zlog.With(). + Str("fleet.agent.id", agent.Id). + Str("fleet.policy.output.name", p.Name).Logger() + + zlog.Info().Msg("preparing elasticsearch output") // The role is required to do api key management if p.Role == nil { @@ -59,27 +63,32 @@ func (p *PolicyOutput) Prepare(ctx context.Context, zlog zerolog.Logger, bulker needNewKey := true switch { case agent.DefaultAPIKey == "": - zlog.Debug().Msg("must generate api key as default API key is not present") + zlog.Info().Msg("must generate api key as default API key is not present") case p.Role.Sha2 != agent.PolicyOutputPermissionsHash: - zlog.Debug().Msg("must generate api key as policy output permissions changed") + zlog.Info().Msg("must generate api key as policy output permissions changed") default: needNewKey = false - zlog.Debug().Msg("policy output permissions are the same") + zlog.Info().Msg("policy output permissions are the same") } if needNewKey { - zlog.Debug(). + zlog.Info(). RawJSON("roles", p.Role.Raw). Str("oldHash", agent.PolicyOutputPermissionsHash). Str("newHash", p.Role.Sha2). Msg("Generating a new API key") + ctx := zlog.WithContext(ctx) outputAPIKey, err := generateOutputAPIKey(ctx, bulker, agent.Id, p.Name, p.Role.Raw) if err != nil { zlog.Error().Err(err).Msg("fail generate output key") return err } + zlog.Info(). + Str(logger.DefaultOutputAPIKeyID+"old", agent.DefaultAPIKey). + Str(logger.DefaultOutputAPIKeyID+".new", outputAPIKey.Agent()). + Msgf("swapping agent API key") agent.DefaultAPIKey = outputAPIKey.Agent() // When a new keys is generated we need to update the Agent record, @@ -128,7 +137,7 @@ func (p *PolicyOutput) Prepare(ctx context.Context, zlog zerolog.Logger, bulker return ErrFailInjectAPIKey } case OutputTypeLogstash: - zlog.Debug().Msg("preparing logstash output") + zlog.Info().Msg("preparing logstash output") zlog.Info().Msg("no actions required for logstash output preparation") default: zlog.Error().Msgf("unknown output type: %s; skipping preparation", p.Type) @@ -160,6 +169,8 @@ func renderUpdatePainlessScript(fields map[string]interface{}) ([]byte, error) { func generateOutputAPIKey(ctx context.Context, bulk bulk.Bulk, agentID, outputName string, roles []byte) (*apikey.APIKey, error) { name := fmt.Sprintf("%s:%s", agentID, outputName) + zerolog.Ctx(ctx).Info().Msgf("generating output API key %s for agent ID %s", + name, agentID) return bulk.APIKeyCreate( ctx, name, From 1404587cf174471d7900d33ff2911baaba0f7462 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 6 Jul 2022 15:52:01 +0200 Subject: [PATCH 26/89] allow to compile a binary for debug (#1559) --- Makefile | 17 ++++++++++++----- README.md | 4 +++- Vagrantfile | 2 +- internal/pkg/rollback/rollback.go | 4 ++-- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 7e1b04ca9..fbcffb3c5 100644 --- a/Makefile +++ b/Makefile @@ -24,13 +24,21 @@ else VERSION=${DEFAULT_VERSION} endif + PLATFORM_TARGETS=$(addprefix release-, $(PLATFORMS)) COMMIT=$(shell git rev-parse --short HEAD) NOW=$(shell date -u '+%Y-%m-%dT%H:%M:%SZ') -LDFLAGS=-w -s -X main.Version=${VERSION} -X main.Commit=${COMMIT} -X main.BuildTime=$(NOW) CMD_COLOR_ON=\033[32m\xE2\x9c\x93 CMD_COLOR_OFF=\033[0m +LDFLAGS=-X main.Version=${VERSION} -X main.Commit=${COMMIT} -X main.BuildTime=$(NOW) +ifeq ($(strip $(DEV)),) +GCFLAGS ?= +LDFLAGS:=-s -w ${LDFLAGS} +else +GCFLAGS ?= all=-N -l +endif + # Directory to dump build tools into GOBIN=$(shell go env GOPATH)/bin/ @@ -46,14 +54,13 @@ list-platforms: ## - Show the possible PLATFORMS .PHONY: local local: ## - Build local binary for local environment (bin/fleet-server) @printf "${CMD_COLOR_ON} Build binaries using local go installation\n${CMD_COLOR_OFF}" - go build -ldflags="${LDFLAGS}" -o ./bin/fleet-server . + go build -gcflags="${GCFLAGS}" -ldflags="${LDFLAGS}" -o ./bin/fleet-server . @printf "${CMD_COLOR_ON} Binaries in ./bin/\n${CMD_COLOR_OFF}" .PHONY: clean clean: ## - Clean up build artifacts @printf "${CMD_COLOR_ON} Clean up build artifacts\n${CMD_COLOR_OFF}" - rm -rf ./bin/ ./build/ - rm .service_token + rm -rf .service_token ./bin/ ./build/ .PHONY: generate generate: ## - Generate schema models @@ -146,7 +153,7 @@ $(PLATFORM_TARGETS): release-%: $(eval $@_GO_ARCH := $(lastword $(subst /, ,$(lastword $(subst release-, ,$@))))) $(eval $@_ARCH := $(TARGET_ARCH_$($@_GO_ARCH))) $(eval $@_BUILDMODE:= $(BUILDMODE_$($@_OS)_$($@_GO_ARCH))) - GOOS=$($@_OS) GOARCH=$($@_GO_ARCH) go build -ldflags="${LDFLAGS}" $($@_BUILDMODE) -o build/binaries/fleet-server-$(VERSION)-$($@_OS)-$($@_ARCH)/fleet-server . + GOOS=$($@_OS) GOARCH=$($@_GO_ARCH) go build -gcflags="${GCFLAGS}" -ldflags="${LDFLAGS}" $($@_BUILDMODE) -o build/binaries/fleet-server-$(VERSION)-$($@_OS)-$($@_ARCH)/fleet-server . @$(MAKE) OS=$($@_OS) ARCH=$($@_ARCH) package-target .PHONY: package-target diff --git a/README.md b/README.md index e4e6aaa73..1ca907f1a 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,10 @@ SNAPSHOT=true EXTERNAL=true PLATFORMS="linux/amd64" PACKAGES="tar.gz" mage -v de Change `release-linux/amd64` to `release-YOUR_OS/platform`. Run `make list-platforms` to check out the possible values. +The `DEV=true` will allow the binary to be debugged ~~with a debugger~~. + ```shell -SNAPSHOT=true make release-linux/amd64 +DEV=true SNAPSHOT=true make release-linux/amd64 vagrant up vagrant ssh diff --git a/Vagrantfile b/Vagrantfile index dc0643c4d..81722c011 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -8,7 +8,7 @@ Vagrant.configure("2") do |config| end config.vm.define "fleet-dev" do |nodeconfig| - nodeconfig.vm.box = "ubuntu/impish64" + nodeconfig.vm.box = "ubuntu/jammy64" nodeconfig.vm.hostname = "fleet-server-dev" diff --git a/internal/pkg/rollback/rollback.go b/internal/pkg/rollback/rollback.go index 2dc5ea176..89ff2e771 100644 --- a/internal/pkg/rollback/rollback.go +++ b/internal/pkg/rollback/rollback.go @@ -43,12 +43,12 @@ func (r *Rollback) Rollback(ctx context.Context) (err error) { log := r.log.With().Str("rollback_fn_name", rb.name).Logger() log.Debug().Msg("rollback function called") if rerr := rb.fn(ctx); rerr != nil { - log.Error().Err(rerr).Msgf("rollback function \"%s\" failed", rb.name) + log.Error().Err(rerr).Msgf("rollback function %q failed", rb.name) if err == nil { err = rerr } } else { - log.Debug().Msgf("rollback function \"%s\" succeeded", rb.name) + log.Debug().Msgf("rollback function %q succeeded", rb.name) } } return //nolint:nakedret // short function From eb581604917d5d4bc1703bdaa6cd6a758d8445e5 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 14 Jul 2022 17:05:41 +0200 Subject: [PATCH 27/89] wip --- 0.notes.md | 1194 ++++++++++++++++++++++++++ internal/pkg/api/handleCheckin.go | 1 - internal/pkg/policy/policy_output.go | 28 +- 3 files changed, 1213 insertions(+), 10 deletions(-) create mode 100644 0.notes.md diff --git a/0.notes.md b/0.notes.md new file mode 100644 index 000000000..e64880c8c --- /dev/null +++ b/0.notes.md @@ -0,0 +1,1194 @@ +Thu 14 Jul 16:09:41 CEST 2022 + +so far I could drill the problem down to how the output permissions change. +It seems that depending on the added/removed/changed integration it might or might +not affect the default output (the output for the agent monitoring logs and metrics) + + - if it affects both (I'm working only with 2 outputs): all good. 2 different apy keys are generated + - if it affects only one (here the not default one) output: the problem happens. +Only one new API key is generated, and both outputs use the same, which does not have enough permissions for both outputs + +#### Only one API key being generated +Endpoint security was removed + + - Elastic-Agent inspect +```text +outputs: + 6d0e50a0-0338-11ed-849a-2dafbb876867: + api_key: KRkB_YEBMKjCD54NTy4Q:HomRenCCSXqdIaPKQdHgNQ + bulk_max_size: 250 + hosts: + - https://192.168.56.1:9200 + ssl: + ca_trusted_fingerprint: 7a145ed00941a323a0d18351e179319a107ee58e07cd7165c72d087b9d685c1f + type: elasticsearch + workers: 8 + default: + api_key: KRkB_YEBMKjCD54NTy4Q:HomRenCCSXqdIaPKQdHgNQ + hosts: + - https://192.168.56.1:9200 + ssl: + ca_trusted_fingerprint: 7a145ed00941a323a0d18351e179319a107ee58e07cd7165c72d087b9d685c1f + type: elasticsearch +revision: 18 +``` + - Fleet-server logs +```json +[{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"zhkB_YEBMKjCD54NpDjo","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[55539],"message":["Invalidate old API keys"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:59:12.000Z"],"@timestamp":["2022-07-14T13:59:06.732Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"ids":["Cxn9_IEBMKjCD54N3RVD"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807146732]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"shkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[53496],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["preparing elasticsearch output"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.681Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130681]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"sxkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[54017],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["policy output permissions are the same"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.681Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130681]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"tBkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"host.hostname":["fleet-server-dev"],"type":["POLICY_CHANGE"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"timeout":[0],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"createdAt":["2022-07-14T13:58:48.878Z"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"inputType":[""],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"id":["policy:67397b70-0354-11ed-849a-2dafbb876867:18:1"],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[54546],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Action delivered to agent on checkin"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.681Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"ackToken":[""],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130681]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"rhkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"fleet.anderson.default.old.apikey":["Cxn9_IEBMKjCD54N3RVD:8TNfbXQqQf2GWErLbHgKOg"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"fleet.anderson.default.new.apikey":["KRkB_YEBMKjCD54NTy4Q:HomRenCCSXqdIaPKQdHgNQ"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[50786],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.404Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130404]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"rxkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.anderson_old_apikey":["Cxn9_IEBMKjCD54N3RVD:8TNfbXQqQf2GWErLbHgKOg"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"fleet.anderson_new_apikey":["KRkB_YEBMKjCD54NTy4Q:HomRenCCSXqdIaPKQdHgNQ"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[51510],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.404Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130404]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"sBkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[52218],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.404Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130404]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"sRkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"fleet.role.hash.sha256":["3da93ea478a1ae088c235461ae13adac7cbca1db0ff5319c8e3bf9421bae8a90"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"fleet.default.apikey.id":["KRkB_YEBMKjCD54NTy4Q"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[52783],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Updating agent record to pick up default output key."],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.404Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130404]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"rRkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[50221],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.403Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130403]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"qRkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[45913],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["preparing elasticsearch output"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.383Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130383]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"qhkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[46463],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["must generate api key as policy output permissions changed"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.383Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130383]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"rBkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[49552],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["generating output API key b91f9075-5f1b-45f1-949c-d7b852e88b7a:6d0e50a0-0338-11ed-849a-2dafbb876867 for agent ID b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.383Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130383]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"qBkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"oldCoord":[1],"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"nQueued":[1],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"coord":[1],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"rev":[18],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["policy agent monitor"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[45534],"message":["New revision of policy received and added to the queue"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.129Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"oldRev":[17],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130129]}] +``` + - API Key doc on ES +```json +{ + "_index": ".security-7", + "_id": "KRkB_YEBMKjCD54NTy4Q", + "_version": 1, + "_seq_no": 275, + "_primary_term": 1, + "found": true, + "_source": { + "doc_type": "api_key", + "creation_time": 1657807130384, + "expiration_time": null, + "api_key_invalidated": false, + "api_key_hash": "{PBKDF2}10000$cExNjhZ5MHlYi+zD6kMjQDkR7cDsCKlHXaLmnaQExkg=$gGv9yrq+svB2+HBnvYhD90L+X46e2gkpsTy2kQQMyiM=", + "role_descriptors": { + "4e80bb57-d538-48d0-9534-93f292e5fa22": { + "cluster": [], + "indices": [ + { + "names": [ + "logs-aws.vpcflow-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + } + ], + "applications": [], + "run_as": [], + "metadata": {}, + "type": "role" + }, + "_elastic_agent_checks": { + "cluster": [ + "monitor" + ], + "indices": [], + "applications": [], + "run_as": [], + "metadata": {}, + "type": "role" + }, + "a1157d27-35ff-4cb0-a4dc-28e21418ebb9": { + "cluster": [], + "indices": [ + { + "names": [ + "logs-generic-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + } + ], + "applications": [], + "run_as": [], + "metadata": {}, + "type": "role" + }, + "daab270e-6fe8-446e-8176-c877fe9e73da": { + "cluster": [], + "indices": [ + { + "names": [ + "logs-system.auth-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-system.syslog-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-system.application-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-system.security-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-system.system-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.cpu-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.diskio-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.filesystem-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.fsstat-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.load-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.memory-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.network-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.process-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.process.summary-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.socket_summary-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.uptime-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + } + ], + "applications": [], + "run_as": [], + "metadata": {}, + "type": "role" + } + }, + "limited_by_role_descriptors": { + "elastic/fleet-server": { + "cluster": [ + "monitor", + "manage_own_api_key" + ], + "indices": [ + { + "names": [ + "logs-*", + "metrics-*", + "traces-*", + "synthetics-*", + ".logs-endpoint.diagnostic.collection-*", + ".logs-endpoint.action.responses-*" + ], + "privileges": [ + "write", + "create_index", + "auto_configure" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "traces-apm.sampled-*" + ], + "privileges": [ + "read", + "monitor", + "maintenance" + ], + "allow_restricted_indices": false + }, + { + "names": [ + ".fleet-*" + ], + "privileges": [ + "read", + "write", + "monitor", + "create_index", + "auto_configure", + "maintenance" + ], + "allow_restricted_indices": true + } + ], + "applications": [ + { + "application": "kibana-*", + "privileges": [ + "reserved_fleet-setup" + ], + "resources": [ + "*" + ] + } + ], + "run_as": [], + "metadata": {}, + "type": "role" + } + }, + "name": "b91f9075-5f1b-45f1-949c-d7b852e88b7a:6d0e50a0-0338-11ed-849a-2dafbb876867", + "version": 8030199, + "metadata_flattened": { + "agent_id": "b91f9075-5f1b-45f1-949c-d7b852e88b7a", + "managed_by": "fleet-server", + "managed": true, + "type": "output" + }, + "creator": { + "principal": "elastic/fleet-server", + "full_name": "Service account - elastic/fleet-server", + "email": null, + "metadata": { + "_elastic_service_account": true + }, + "realm": "_service_account", + "realm_type": "_service_account" + } + } +} +``` + +#### Both API keys being generated +Endpoint security was added + + - Elastic-Agent inspect +```text +outputs: + 6d0e50a0-0338-11ed-849a-2dafbb876867: + api_key: VBkT_YEBMKjCD54NWmW4:yrIW1wxXSpmwEcIANQ-Krw + bulk_max_size: 250 + hosts: + - https://192.168.56.1:9200 + ssl: + ca_trusted_fingerprint: 7a145ed00941a323a0d18351e179319a107ee58e07cd7165c72d087b9d685c1f + type: elasticsearch + workers: 8 + default: + api_key: VhkT_YEBMKjCD54NW2XR:7wYIks1mTmeacWwho66nJA + hosts: + - https://192.168.56.1:9200 + ssl: + ca_trusted_fingerprint: 7a145ed00941a323a0d18351e179319a107ee58e07cd7165c72d087b9d685c1f + type: elasticsearch +revision: 19 + +``` + +- Fleet-server logs +```json +[{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"-RkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[797],"message":["bb876867\",\"rev\":19,\"coord\":1,\"oldRev\":18,\"oldCoord\":1,\"nQueued\":1,\"fleet.policy.id\":\"67397b70-0354-11ed-849a-2dafbb876867\",\"@timestamp\":\"2022-07-14T14:18:32.745Z\",\"message\":\"New revision of policy received and added to the queue\"}"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:50.393Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808330393]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"DxkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH6Z3QP79ZTPS3DM1MVWKJ"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[18841],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["ack policy"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:42.798Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"nEvents":[1],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808322798]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"DhkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[18638],"message":["Invalidate old API keys"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:41.952Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"ids":["KRkB_YEBMKjCD54NTy4Q","KRkB_YEBMKjCD54NTy4Q"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808321952]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"DRkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"agentId":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH6Z3QP79ZTPS3DM1MVWKJ"],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"actionSubType":["ACKNOWLEDGED"],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"timestamp":["2022-07-14T14:18:41.65523+00:00"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[18117],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["ack event"],"data_stream.type":["logs"],"n":[0],"host.architecture":["x86_64"],"actionType":["ACTION_RESULT"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:41.951Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"actionId":["policy:67397b70-0354-11ed-849a-2dafbb876867:19:1"],"agent.version":["8.3.1"],"host.os.family":["debian"],"nEvents":[1],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808321951]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"DBkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"host.hostname":["fleet-server-dev"],"type":["POLICY_CHANGE"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"timeout":[0],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"createdAt":["2022-07-14T14:18:31.609Z"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"inputType":[""],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"id":["policy:67397b70-0354-11ed-849a-2dafbb876867:19:1"],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[17645],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Action delivered to agent on checkin"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.608Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"ackToken":[""],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313608]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"BxkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[14515],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"CBkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"fleet.anderson.default.old.apikey":["VBkT_YEBMKjCD54NWmW4:yrIW1wxXSpmwEcIANQ-Krw"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"fleet.anderson.default.new.apikey":["VhkT_YEBMKjCD54NW2XR:7wYIks1mTmeacWwho66nJA"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[15051],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"CRkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.anderson_old_apikey":["VBkT_YEBMKjCD54NWmW4:yrIW1wxXSpmwEcIANQ-Krw"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"fleet.anderson_new_apikey":["VhkT_YEBMKjCD54NW2XR:7wYIks1mTmeacWwho66nJA"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[15746],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"ChkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[16425],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"CxkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"fleet.role.hash.sha256":["ac0c515dcc1ef486f784feccdb11c88cd84cd1d2e869a66bc1dad6d29b837a0a"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"fleet.default.apikey.id":["VhkT_YEBMKjCD54NW2XR"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[16961],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Updating agent record to pick up default output key."],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"AxkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[9896],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["preparing elasticsearch output"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.296Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313296]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"BBkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[10417],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["must generate api key as policy output permissions changed"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.296Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313296]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"BhkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[13904],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["generating output API key b91f9075-5f1b-45f1-949c-d7b852e88b7a:default for agent ID b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.296Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313296]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"_hkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[6621],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"_xkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"fleet.anderson.default.old.apikey":["KRkB_YEBMKjCD54NTy4Q:HomRenCCSXqdIaPKQdHgNQ"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"fleet.anderson.default.new.apikey":["VBkT_YEBMKjCD54NWmW4:yrIW1wxXSpmwEcIANQ-Krw"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[7186],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"ABkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.anderson_old_apikey":["KRkB_YEBMKjCD54NTy4Q:HomRenCCSXqdIaPKQdHgNQ"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"fleet.anderson_new_apikey":["VBkT_YEBMKjCD54NWmW4:yrIW1wxXSpmwEcIANQ-Krw"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[7910],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"ARkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[8618],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"AhkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"fleet.role.hash.sha256":["a4af23b737b102e5e4555a6a062531fe08660d096e5c7aa9ed73d61fdf7ff5b6"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"fleet.default.apikey.id":["VBkT_YEBMKjCD54NWmW4"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[9183],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Updating agent record to pick up default output key."],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"-hkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[1028],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["preparing elasticsearch output"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.015Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313015]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"-xkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[1578],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["must generate api key as policy output permissions changed"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.015Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313015]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"_RkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[5952],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["generating output API key b91f9075-5f1b-45f1-949c-d7b852e88b7a:6d0e50a0-0338-11ed-849a-2dafbb876867 for agent ID b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.015Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313015]}] +``` + +- API Key docs on ES + + - output 6d0e50a0-0338-11ed-849a-2dafbb876867, key id VBkT_YEBMKjCD54NWmW4 +```json +{ + "_index": ".security-7", + "_id": "VBkT_YEBMKjCD54NWmW4", + "_version": 1, + "_seq_no": 277, + "_primary_term": 1, + "found": true, + "_source": { + "doc_type": "api_key", + "creation_time": 1657808313016, + "expiration_time": null, + "api_key_invalidated": false, + "api_key_hash": "{PBKDF2}10000$jrCIMrbr1+zDVZw363AkxtnkkpavDr/SNgtr2wiU69Q=$Z5bwX7LA0fVmq99KwaMEDSqaYIU8lwhD1LTqct5etAw=", + "role_descriptors": { + "4e80bb57-d538-48d0-9534-93f292e5fa22": { + "cluster": [], + "indices": [ + { + "names": [ + "logs-aws.vpcflow-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + } + ], + "applications": [], + "run_as": [], + "metadata": {}, + "type": "role" + }, + "6d72b1a4-dc33-4d42-b92a-744bb1ef364e": { + "cluster": [], + "indices": [ + { + "names": [ + ".logs-endpoint.action.responses-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + ".logs-endpoint.actions-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-endpoint.alerts-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + ".logs-endpoint.diagnostic.collection-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-endpoint.events.file-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-endpoint.events.library-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-endpoint.metadata-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-endpoint.metrics-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-endpoint.events.network-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-endpoint.policy-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-endpoint.events.process-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-endpoint.events.registry-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-endpoint.events.security-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + } + ], + "applications": [], + "run_as": [], + "metadata": {}, + "type": "role" + }, + "_elastic_agent_checks": { + "cluster": [ + "monitor" + ], + "indices": [], + "applications": [], + "run_as": [], + "metadata": {}, + "type": "role" + }, + "a1157d27-35ff-4cb0-a4dc-28e21418ebb9": { + "cluster": [], + "indices": [ + { + "names": [ + "logs-generic-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + } + ], + "applications": [], + "run_as": [], + "metadata": {}, + "type": "role" + }, + "daab270e-6fe8-446e-8176-c877fe9e73da": { + "cluster": [], + "indices": [ + { + "names": [ + "logs-system.auth-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-system.syslog-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-system.application-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-system.security-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-system.system-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.cpu-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.diskio-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.filesystem-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.fsstat-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.load-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.memory-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.network-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.process-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.process.summary-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.socket_summary-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-system.uptime-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + } + ], + "applications": [], + "run_as": [], + "metadata": {}, + "type": "role" + } + }, + "limited_by_role_descriptors": { + "elastic/fleet-server": { + "cluster": [ + "monitor", + "manage_own_api_key" + ], + "indices": [ + { + "names": [ + "logs-*", + "metrics-*", + "traces-*", + "synthetics-*", + ".logs-endpoint.diagnostic.collection-*", + ".logs-endpoint.action.responses-*" + ], + "privileges": [ + "write", + "create_index", + "auto_configure" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "traces-apm.sampled-*" + ], + "privileges": [ + "read", + "monitor", + "maintenance" + ], + "allow_restricted_indices": false + }, + { + "names": [ + ".fleet-*" + ], + "privileges": [ + "read", + "write", + "monitor", + "create_index", + "auto_configure", + "maintenance" + ], + "allow_restricted_indices": true + } + ], + "applications": [ + { + "application": "kibana-*", + "privileges": [ + "reserved_fleet-setup" + ], + "resources": [ + "*" + ] + } + ], + "run_as": [], + "metadata": {}, + "type": "role" + } + }, + "name": "b91f9075-5f1b-45f1-949c-d7b852e88b7a:6d0e50a0-0338-11ed-849a-2dafbb876867", + "version": 8030199, + "metadata_flattened": { + "agent_id": "b91f9075-5f1b-45f1-949c-d7b852e88b7a", + "managed_by": "fleet-server", + "managed": true, + "type": "output" + }, + "creator": { + "principal": "elastic/fleet-server", + "full_name": "Service account - elastic/fleet-server", + "email": null, + "metadata": { + "_elastic_service_account": true + }, + "realm": "_service_account", + "realm_type": "_service_account" + } + } +} +``` + + - output default, key id VhkT_YEBMKjCD54NW2XR +```json +{ + "_index": ".security-7", + "_id": "VhkT_YEBMKjCD54NW2XR", + "_version": 1, + "_seq_no": 278, + "_primary_term": 1, + "found": true, + "_source": { + "doc_type": "api_key", + "creation_time": 1657808313298, + "expiration_time": null, + "api_key_invalidated": false, + "api_key_hash": "{PBKDF2}10000$+/HtACFwZUwk/31JJw7j2Lk+aQtvmlIsdvUzqHMF/Pw=$3CVORNqyrjhg0Xvfdd6k5MXOWmS1Cf/d9QvnOXiXTkM=", + "role_descriptors": { + "_elastic_agent_checks": { + "cluster": [ + "monitor" + ], + "indices": [], + "applications": [], + "run_as": [], + "metadata": {}, + "type": "role" + }, + "_elastic_agent_monitoring": { + "cluster": [], + "indices": [ + { + "names": [ + "logs-elastic_agent.apm_server-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-elastic_agent.apm_server-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-elastic_agent.auditbeat-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-elastic_agent.auditbeat-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-elastic_agent.cloudbeat-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-elastic_agent.cloudbeat-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-elastic_agent-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-elastic_agent.elastic_agent-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-elastic_agent.endpoint_security-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-elastic_agent.endpoint_security-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-elastic_agent.filebeat-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-elastic_agent.filebeat-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-elastic_agent.fleet_server-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-elastic_agent.fleet_server-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-elastic_agent.heartbeat-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-elastic_agent.heartbeat-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-elastic_agent.metricbeat-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-elastic_agent.metricbeat-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-elastic_agent.osquerybeat-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-elastic_agent.osquerybeat-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "logs-elastic_agent.packetbeat-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "metrics-elastic_agent.packetbeat-default" + ], + "privileges": [ + "auto_configure", + "create_doc" + ], + "allow_restricted_indices": false + } + ], + "applications": [], + "run_as": [], + "metadata": {}, + "type": "role" + } + }, + "limited_by_role_descriptors": { + "elastic/fleet-server": { + "cluster": [ + "monitor", + "manage_own_api_key" + ], + "indices": [ + { + "names": [ + "logs-*", + "metrics-*", + "traces-*", + "synthetics-*", + ".logs-endpoint.diagnostic.collection-*", + ".logs-endpoint.action.responses-*" + ], + "privileges": [ + "write", + "create_index", + "auto_configure" + ], + "allow_restricted_indices": false + }, + { + "names": [ + "traces-apm.sampled-*" + ], + "privileges": [ + "read", + "monitor", + "maintenance" + ], + "allow_restricted_indices": false + }, + { + "names": [ + ".fleet-*" + ], + "privileges": [ + "read", + "write", + "monitor", + "create_index", + "auto_configure", + "maintenance" + ], + "allow_restricted_indices": true + } + ], + "applications": [ + { + "application": "kibana-*", + "privileges": [ + "reserved_fleet-setup" + ], + "resources": [ + "*" + ] + } + ], + "run_as": [], + "metadata": {}, + "type": "role" + } + }, + "name": "b91f9075-5f1b-45f1-949c-d7b852e88b7a:default", + "version": 8030199, + "metadata_flattened": { + "agent_id": "b91f9075-5f1b-45f1-949c-d7b852e88b7a", + "managed_by": "fleet-server", + "managed": true, + "type": "output" + }, + "creator": { + "principal": "elastic/fleet-server", + "full_name": "Service account - elastic/fleet-server", + "email": null, + "metadata": { + "_elastic_service_account": true + }, + "realm": "_service_account", + "realm_type": "_service_account" + } + } +} +``` diff --git a/internal/pkg/api/handleCheckin.go b/internal/pkg/api/handleCheckin.go index a5e3c97f2..f95c0769f 100644 --- a/internal/pkg/api/handleCheckin.go +++ b/internal/pkg/api/handleCheckin.go @@ -446,7 +446,6 @@ func processPolicy(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, a // Parse the outputs maps in order to prepare the outputs const outputsProperty = "outputs" outputs, err := smap.Parse(pp.Fields[outputsProperty]) - if err != nil { return nil, err } diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index e55ae9441..b172bd73b 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -49,7 +49,9 @@ func (p *PolicyOutput) Prepare(ctx context.Context, zlog zerolog.Logger, bulker // The role is required to do api key management if p.Role == nil { - zlog.Error().Str("name", p.Name).Msg("policy does not contain required output permission section") + zlog.Error(). + Str("fleet.output.name", p.Name). + Msg("policy does not contain required output permission section") return ErrNoOutputPerms } @@ -84,18 +86,17 @@ func (p *PolicyOutput) Prepare(ctx context.Context, zlog zerolog.Logger, bulker zlog.Error().Err(err).Msg("fail generate output key") return err } - zlog.Info(). - Str(logger.DefaultOutputAPIKeyID+"old", agent.DefaultAPIKey). - Str(logger.DefaultOutputAPIKeyID+".new", outputAPIKey.Agent()). - Msgf("swapping agent API key") + Str("fleet.anderson.default.old.apikey", agent.DefaultAPIKey). + Str("fleet.anderson.default.new.apikey", outputAPIKey.Agent()). + Msg("setting / swapping agent default API key") agent.DefaultAPIKey = outputAPIKey.Agent() // When a new keys is generated we need to update the Agent record, // this will need to be updated when multiples Elasticsearch output // are used. zlog.Info(). - Str("hash.sha256", p.Role.Sha2). + Str("fleet.role.hash.sha256", p.Role.Sha2). Str(logger.DefaultOutputAPIKeyID, outputAPIKey.ID). Msg("Updating agent record to pick up default output key.") @@ -113,14 +114,13 @@ func (p *PolicyOutput) Prepare(ctx context.Context, zlog zerolog.Logger, bulker // Using painless script to append the old keys to the history body, err := renderUpdatePainlessScript(fields) - if err != nil { - return err + return fmt.Errorf("could no tupdate painless script: %w", err) } if err = bulker.Update(ctx, dl.FleetAgents, agent.Id, body); err != nil { zlog.Error().Err(err).Msg("fail update agent record") - return err + return fmt.Errorf("fail update agent record: %w", err) } } @@ -133,6 +133,16 @@ func (p *PolicyOutput) Prepare(ctx context.Context, zlog zerolog.Logger, bulker // in place to reduce number of agent policy allocation when sending the updated // agent policy to multiple agents. // See: https://github.com/elastic/fleet-server/issues/1301 + // + // WIP: + // The agent struct is a pointer, thus shared between the runs of Prepare + // for each output. Given 2 outputs (1 and 2), if 1 gets a new API key, + // agent.DefaultAPIKey changes. However, if 2 does not get one, the code + // below runs anyway, therefore the API key for output 2 will be set to the + // api key for output 1 + // The agent struct cannot be shared! The new API key must be its own local + // variable and a special case for the 'default output' has to handle + // updating agent.DefaultAPIKey. if ok := setMapObj(outputMap, agent.DefaultAPIKey, p.Name, "api_key"); !ok { return ErrFailInjectAPIKey } From d2ec5277ca95a152d70f46b606312d351a0fe883 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 14 Jul 2022 17:12:42 +0200 Subject: [PATCH 28/89] add affected versions --- 0.notes.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/0.notes.md b/0.notes.md index e64880c8c..7467a1f88 100644 --- a/0.notes.md +++ b/0.notes.md @@ -8,6 +8,9 @@ not affect the default output (the output for the agent monitoring logs and metr - if it affects only one (here the not default one) output: the problem happens. Only one new API key is generated, and both outputs use the same, which does not have enough permissions for both outputs +From git blame, the commit causing this problem was merged on 04/04/2022 and +backported to v8.2.0 and v8.3.0 + #### Only one API key being generated Endpoint security was removed From b463a772c1193ed622f2c241807e2871d55a21b5 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 14 Jul 2022 17:26:24 +0200 Subject: [PATCH 29/89] evidence of the problem --- 0.notes.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/0.notes.md b/0.notes.md index 7467a1f88..fdacdac8c 100644 --- a/0.notes.md +++ b/0.notes.md @@ -8,6 +8,13 @@ not affect the default output (the output for the agent monitoring logs and metr - if it affects only one (here the not default one) output: the problem happens. Only one new API key is generated, and both outputs use the same, which does not have enough permissions for both outputs +One thing that seems to indicate the problem happened is the log `Invalidate old API keys` +where the `ids` field has got repeated key IDs, like this one: +```json +[{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"DhkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[18638],"message":["Invalidate old API keys"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:41.952Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"ids":["KRkB_YEBMKjCD54NTy4Q","KRkB_YEBMKjCD54NTy4Q"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808321952]}] +``` + + From git blame, the commit causing this problem was merged on 04/04/2022 and backported to v8.2.0 and v8.3.0 From 91c78f2faf4d4f732f91ab9f91d3119c26fc1f1c Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 18 Jul 2022 19:47:25 +0200 Subject: [PATCH 30/89] better reading layout --- 0.notes.md | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/0.notes.md b/0.notes.md index fdacdac8c..36c955a25 100644 --- a/0.notes.md +++ b/0.notes.md @@ -15,17 +15,14 @@ where the `ids` field has got repeated key IDs, like this one: ``` -From git blame, the commit causing this problem was merged on 04/04/2022 and -backported to v8.2.0 and v8.3.0 - #### Only one API key being generated Endpoint security was removed - - Elastic-Agent inspect +- Elastic-Agent inspect ```text outputs: 6d0e50a0-0338-11ed-849a-2dafbb876867: - api_key: KRkB_YEBMKjCD54NTy4Q:HomRenCCSXqdIaPKQdHgNQ + api_key: KRkB_YEBMKjCD54NTy4Q: bulk_max_size: 250 hosts: - https://192.168.56.1:9200 @@ -34,7 +31,7 @@ outputs: type: elasticsearch workers: 8 default: - api_key: KRkB_YEBMKjCD54NTy4Q:HomRenCCSXqdIaPKQdHgNQ + api_key: KRkB_YEBMKjCD54NTy4Q: hosts: - https://192.168.56.1:9200 ssl: @@ -42,11 +39,17 @@ outputs: type: elasticsearch revision: 18 ``` - - Fleet-server logs +
+ Fleet-server logs (click to expand) + ```json -[{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"zhkB_YEBMKjCD54NpDjo","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[55539],"message":["Invalidate old API keys"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:59:12.000Z"],"@timestamp":["2022-07-14T13:59:06.732Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"ids":["Cxn9_IEBMKjCD54N3RVD"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807146732]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"shkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[53496],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["preparing elasticsearch output"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.681Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130681]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"sxkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[54017],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["policy output permissions are the same"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.681Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130681]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"tBkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"host.hostname":["fleet-server-dev"],"type":["POLICY_CHANGE"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"timeout":[0],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"createdAt":["2022-07-14T13:58:48.878Z"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"inputType":[""],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"id":["policy:67397b70-0354-11ed-849a-2dafbb876867:18:1"],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[54546],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Action delivered to agent on checkin"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.681Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"ackToken":[""],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130681]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"rhkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"fleet.anderson.default.old.apikey":["Cxn9_IEBMKjCD54N3RVD:8TNfbXQqQf2GWErLbHgKOg"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"fleet.anderson.default.new.apikey":["KRkB_YEBMKjCD54NTy4Q:HomRenCCSXqdIaPKQdHgNQ"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[50786],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.404Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130404]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"rxkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.anderson_old_apikey":["Cxn9_IEBMKjCD54N3RVD:8TNfbXQqQf2GWErLbHgKOg"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"fleet.anderson_new_apikey":["KRkB_YEBMKjCD54NTy4Q:HomRenCCSXqdIaPKQdHgNQ"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[51510],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.404Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130404]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"sBkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[52218],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.404Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130404]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"sRkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"fleet.role.hash.sha256":["3da93ea478a1ae088c235461ae13adac7cbca1db0ff5319c8e3bf9421bae8a90"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"fleet.default.apikey.id":["KRkB_YEBMKjCD54NTy4Q"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[52783],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Updating agent record to pick up default output key."],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.404Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130404]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"rRkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[50221],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.403Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130403]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"qRkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[45913],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["preparing elasticsearch output"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.383Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130383]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"qhkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[46463],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["must generate api key as policy output permissions changed"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.383Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130383]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"rBkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[49552],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["generating output API key b91f9075-5f1b-45f1-949c-d7b852e88b7a:6d0e50a0-0338-11ed-849a-2dafbb876867 for agent ID b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.383Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130383]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"qBkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"oldCoord":[1],"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"nQueued":[1],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"coord":[1],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"rev":[18],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["policy agent monitor"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[45534],"message":["New revision of policy received and added to the queue"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.129Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"oldRev":[17],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130129]}] +[{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"zhkB_YEBMKjCD54NpDjo","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[55539],"message":["Invalidate old API keys"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:59:12.000Z"],"@timestamp":["2022-07-14T13:59:06.732Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"ids":["Cxn9_IEBMKjCD54N3RVD"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807146732]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"shkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[53496],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["preparing elasticsearch output"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.681Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130681]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"sxkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[54017],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["policy output permissions are the same"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.681Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130681]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"tBkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"host.hostname":["fleet-server-dev"],"type":["POLICY_CHANGE"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"timeout":[0],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"createdAt":["2022-07-14T13:58:48.878Z"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"inputType":[""],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"id":["policy:67397b70-0354-11ed-849a-2dafbb876867:18:1"],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[54546],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Action delivered to agent on checkin"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.681Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"ackToken":[""],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130681]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"rhkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"fleet.anderson.default.old.apikey":["Cxn9_IEBMKjCD54N3RVD:8TNfbXQqQf2GWErLbHgKOg"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"fleet.anderson.default.new.apikey":["KRkB_YEBMKjCD54NTy4Q:"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[50786],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.404Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130404]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"rxkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.anderson_old_apikey":["Cxn9_IEBMKjCD54N3RVD:8TNfbXQqQf2GWErLbHgKOg"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"fleet.anderson_new_apikey":["KRkB_YEBMKjCD54NTy4Q:"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[51510],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.404Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130404]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"sBkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[52218],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.404Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130404]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"sRkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"fleet.role.hash.sha256":["3da93ea478a1ae088c235461ae13adac7cbca1db0ff5319c8e3bf9421bae8a90"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"fleet.default.apikey.id":["KRkB_YEBMKjCD54NTy4Q"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[52783],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Updating agent record to pick up default output key."],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.404Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130404]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"rRkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[50221],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.403Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130403]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"qRkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[45913],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["preparing elasticsearch output"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.383Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130383]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"qhkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[46463],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["must generate api key as policy output permissions changed"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.383Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130383]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"rBkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[49552],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["generating output API key b91f9075-5f1b-45f1-949c-d7b852e88b7a:6d0e50a0-0338-11ed-849a-2dafbb876867 for agent ID b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.383Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130383]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"qBkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"oldCoord":[1],"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"nQueued":[1],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"coord":[1],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"rev":[18],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["policy agent monitor"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[45534],"message":["New revision of policy received and added to the queue"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.129Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"oldRev":[17],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130129]}] ``` - - API Key doc on ES +
+ +
+ API Key doc on ES (click to expand) + ```json { "_index": ".security-7", @@ -366,15 +369,16 @@ revision: 18 } } ``` +
#### Both API keys being generated Endpoint security was added - - Elastic-Agent inspect +- Elastic-Agent inspect ```text outputs: 6d0e50a0-0338-11ed-849a-2dafbb876867: - api_key: VBkT_YEBMKjCD54NWmW4:yrIW1wxXSpmwEcIANQ-Krw + api_key: VBkT_YEBMKjCD54NWmW4: bulk_max_size: 250 hosts: - https://192.168.56.1:9200 @@ -383,7 +387,7 @@ outputs: type: elasticsearch workers: 8 default: - api_key: VhkT_YEBMKjCD54NW2XR:7wYIks1mTmeacWwho66nJA + api_key: VhkT_YEBMKjCD54NW2XR: hosts: - https://192.168.56.1:9200 ssl: @@ -393,14 +397,18 @@ revision: 19 ``` -- Fleet-server logs +
+ Fleet-server logs (click to expand) + ```json -[{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"-RkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[797],"message":["bb876867\",\"rev\":19,\"coord\":1,\"oldRev\":18,\"oldCoord\":1,\"nQueued\":1,\"fleet.policy.id\":\"67397b70-0354-11ed-849a-2dafbb876867\",\"@timestamp\":\"2022-07-14T14:18:32.745Z\",\"message\":\"New revision of policy received and added to the queue\"}"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:50.393Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808330393]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"DxkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH6Z3QP79ZTPS3DM1MVWKJ"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[18841],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["ack policy"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:42.798Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"nEvents":[1],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808322798]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"DhkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[18638],"message":["Invalidate old API keys"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:41.952Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"ids":["KRkB_YEBMKjCD54NTy4Q","KRkB_YEBMKjCD54NTy4Q"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808321952]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"DRkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"agentId":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH6Z3QP79ZTPS3DM1MVWKJ"],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"actionSubType":["ACKNOWLEDGED"],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"timestamp":["2022-07-14T14:18:41.65523+00:00"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[18117],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["ack event"],"data_stream.type":["logs"],"n":[0],"host.architecture":["x86_64"],"actionType":["ACTION_RESULT"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:41.951Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"actionId":["policy:67397b70-0354-11ed-849a-2dafbb876867:19:1"],"agent.version":["8.3.1"],"host.os.family":["debian"],"nEvents":[1],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808321951]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"DBkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"host.hostname":["fleet-server-dev"],"type":["POLICY_CHANGE"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"timeout":[0],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"createdAt":["2022-07-14T14:18:31.609Z"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"inputType":[""],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"id":["policy:67397b70-0354-11ed-849a-2dafbb876867:19:1"],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[17645],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Action delivered to agent on checkin"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.608Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"ackToken":[""],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313608]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"BxkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[14515],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"CBkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"fleet.anderson.default.old.apikey":["VBkT_YEBMKjCD54NWmW4:yrIW1wxXSpmwEcIANQ-Krw"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"fleet.anderson.default.new.apikey":["VhkT_YEBMKjCD54NW2XR:7wYIks1mTmeacWwho66nJA"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[15051],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"CRkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.anderson_old_apikey":["VBkT_YEBMKjCD54NWmW4:yrIW1wxXSpmwEcIANQ-Krw"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"fleet.anderson_new_apikey":["VhkT_YEBMKjCD54NW2XR:7wYIks1mTmeacWwho66nJA"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[15746],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"ChkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[16425],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"CxkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"fleet.role.hash.sha256":["ac0c515dcc1ef486f784feccdb11c88cd84cd1d2e869a66bc1dad6d29b837a0a"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"fleet.default.apikey.id":["VhkT_YEBMKjCD54NW2XR"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[16961],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Updating agent record to pick up default output key."],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"AxkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[9896],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["preparing elasticsearch output"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.296Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313296]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"BBkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[10417],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["must generate api key as policy output permissions changed"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.296Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313296]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"BhkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[13904],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["generating output API key b91f9075-5f1b-45f1-949c-d7b852e88b7a:default for agent ID b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.296Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313296]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"_hkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[6621],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"_xkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"fleet.anderson.default.old.apikey":["KRkB_YEBMKjCD54NTy4Q:HomRenCCSXqdIaPKQdHgNQ"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"fleet.anderson.default.new.apikey":["VBkT_YEBMKjCD54NWmW4:yrIW1wxXSpmwEcIANQ-Krw"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[7186],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"ABkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.anderson_old_apikey":["KRkB_YEBMKjCD54NTy4Q:HomRenCCSXqdIaPKQdHgNQ"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"fleet.anderson_new_apikey":["VBkT_YEBMKjCD54NWmW4:yrIW1wxXSpmwEcIANQ-Krw"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[7910],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"ARkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[8618],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"AhkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"fleet.role.hash.sha256":["a4af23b737b102e5e4555a6a062531fe08660d096e5c7aa9ed73d61fdf7ff5b6"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"fleet.default.apikey.id":["VBkT_YEBMKjCD54NWmW4"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[9183],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Updating agent record to pick up default output key."],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"-hkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[1028],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["preparing elasticsearch output"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.015Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313015]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"-xkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[1578],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["must generate api key as policy output permissions changed"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.015Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313015]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"_RkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[5952],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["generating output API key b91f9075-5f1b-45f1-949c-d7b852e88b7a:6d0e50a0-0338-11ed-849a-2dafbb876867 for agent ID b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.015Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313015]}] +[{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"-RkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[797],"message":["bb876867\",\"rev\":19,\"coord\":1,\"oldRev\":18,\"oldCoord\":1,\"nQueued\":1,\"fleet.policy.id\":\"67397b70-0354-11ed-849a-2dafbb876867\",\"@timestamp\":\"2022-07-14T14:18:32.745Z\",\"message\":\"New revision of policy received and added to the queue\"}"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:50.393Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808330393]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"DxkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH6Z3QP79ZTPS3DM1MVWKJ"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[18841],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["ack policy"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:42.798Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"nEvents":[1],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808322798]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"DhkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[18638],"message":["Invalidate old API keys"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:41.952Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"ids":["KRkB_YEBMKjCD54NTy4Q","KRkB_YEBMKjCD54NTy4Q"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808321952]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"DRkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"agentId":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH6Z3QP79ZTPS3DM1MVWKJ"],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"actionSubType":["ACKNOWLEDGED"],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"timestamp":["2022-07-14T14:18:41.65523+00:00"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[18117],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["ack event"],"data_stream.type":["logs"],"n":[0],"host.architecture":["x86_64"],"actionType":["ACTION_RESULT"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:41.951Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"actionId":["policy:67397b70-0354-11ed-849a-2dafbb876867:19:1"],"agent.version":["8.3.1"],"host.os.family":["debian"],"nEvents":[1],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808321951]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"DBkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"host.hostname":["fleet-server-dev"],"type":["POLICY_CHANGE"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"timeout":[0],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"createdAt":["2022-07-14T14:18:31.609Z"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"inputType":[""],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"id":["policy:67397b70-0354-11ed-849a-2dafbb876867:19:1"],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[17645],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Action delivered to agent on checkin"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.608Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"ackToken":[""],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313608]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"BxkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[14515],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"CBkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"fleet.anderson.default.old.apikey":["VBkT_YEBMKjCD54NWmW4:"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"fleet.anderson.default.new.apikey":["VhkT_YEBMKjCD54NW2XR:"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[15051],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"CRkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.anderson_old_apikey":["VBkT_YEBMKjCD54NWmW4:"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"fleet.anderson_new_apikey":["VhkT_YEBMKjCD54NW2XR:"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[15746],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"ChkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[16425],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"CxkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"fleet.role.hash.sha256":["ac0c515dcc1ef486f784feccdb11c88cd84cd1d2e869a66bc1dad6d29b837a0a"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"fleet.default.apikey.id":["VhkT_YEBMKjCD54NW2XR"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[16961],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Updating agent record to pick up default output key."],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"AxkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[9896],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["preparing elasticsearch output"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.296Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313296]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"BBkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[10417],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["must generate api key as policy output permissions changed"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.296Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313296]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"BhkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[13904],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["generating output API key b91f9075-5f1b-45f1-949c-d7b852e88b7a:default for agent ID b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.296Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313296]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"_hkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[6621],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"_xkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"fleet.anderson.default.old.apikey":["KRkB_YEBMKjCD54NTy4Q:"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"fleet.anderson.default.new.apikey":["VBkT_YEBMKjCD54NWmW4:"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[7186],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"ABkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.anderson_old_apikey":["KRkB_YEBMKjCD54NTy4Q:"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"fleet.anderson_new_apikey":["VBkT_YEBMKjCD54NWmW4:"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[7910],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"ARkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[8618],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"AhkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"fleet.role.hash.sha256":["a4af23b737b102e5e4555a6a062531fe08660d096e5c7aa9ed73d61fdf7ff5b6"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"fleet.default.apikey.id":["VBkT_YEBMKjCD54NWmW4"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[9183],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Updating agent record to pick up default output key."],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"-hkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[1028],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["preparing elasticsearch output"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.015Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313015]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"-xkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[1578],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["must generate api key as policy output permissions changed"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.015Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313015]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"_RkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[5952],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["generating output API key b91f9075-5f1b-45f1-949c-d7b852e88b7a:6d0e50a0-0338-11ed-849a-2dafbb876867 for agent ID b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.015Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313015]}] ``` +
-- API Key docs on ES +
+ API Key docs on ES (click to expand) - - output 6d0e50a0-0338-11ed-849a-2dafbb876867, key id VBkT_YEBMKjCD54NWmW4 +- output 6d0e50a0-0338-11ed-849a-2dafbb876867, key id VBkT_YEBMKjCD54NWmW4 ```json { "_index": ".security-7", @@ -859,8 +867,11 @@ revision: 19 } } ``` +
+ +
+ output default, key id VhkT_YEBMKjCD54NW2XR (click to expand) - - output default, key id VhkT_YEBMKjCD54NW2XR ```json { "_index": ".security-7", @@ -1202,3 +1213,5 @@ revision: 19 } } ``` + +
\ No newline at end of file From 008f2732f5eccd8f1e75e085f29268ef285f9c17 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 21 Jul 2022 11:07:16 +0200 Subject: [PATCH 31/89] add elasticsearchOutputs to agent schema --- internal/pkg/model/schema.go | 42 +++++++++++++++++++++----- model/schema.json | 57 +++++++++++++++++++++++++++++++++--- 2 files changed, 87 insertions(+), 12 deletions(-) diff --git a/internal/pkg/model/schema.go b/internal/pkg/model/schema.go index fca90db0b..086b044f1 100644 --- a/internal/pkg/model/schema.go +++ b/internal/pkg/model/schema.go @@ -1,7 +1,3 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - // Code generated by schema-generate. DO NOT EDIT. package model @@ -29,6 +25,16 @@ func (d *ESDocument) ESInitialize(id string, seqno, version int64) { d.Version = version } +// APIKeyHistoryItems +type APIKeyHistoryItems struct { + + // API Key identifier + ID string `json:"id,omitempty"` + + // Date/time the API key was retired + RetiredAt string `json:"retired_at,omitempty"` +} + // Action An Elastic Agent action type Action struct { ESDocument @@ -124,15 +130,18 @@ type Agent struct { Active bool `json:"active"` Agent *AgentMetadata `json:"agent,omitempty"` - // API key the Elastic Agent uses to authenticate with elasticsearch + // Deprecated. Use Outputs instead. API key the Elastic Agent uses to authenticate with elasticsearch DefaultAPIKey string `json:"default_api_key,omitempty"` - // Default API Key History + // Deprecated. Use Outputs instead. Default API Key History DefaultAPIKeyHistory []DefaultAPIKeyHistoryItems `json:"default_api_key_history,omitempty"` - // ID of the API key the Elastic Agent uses to authenticate with elasticsearch + // Deprecated. Use Outputs instead. ID of the API key the Elastic Agent uses to authenticate with elasticsearch DefaultAPIKeyID string `json:"default_api_key_id,omitempty"` + // ElasticsearchOutputs is the policy output data for each Elasticsearch output. It maps the output name to its data + ElasticsearchOutputs map[string]*PolicyOutput `json:"elasticsearch_outputs,omitempty"` + // Date/time the Elastic Agent enrolled EnrolledAt string `json:"enrolled_at"` @@ -157,7 +166,7 @@ type Agent struct { // The policy ID for the Elastic Agent PolicyID string `json:"policy_id,omitempty"` - // The policy output permissions hash + // Deprecated. Use Outputs instead. The policy output permissions hash PolicyOutputPermissionsHash string `json:"policy_output_permissions_hash,omitempty"` // The current policy revision_idx for the Elastic Agent @@ -333,6 +342,23 @@ type PolicyLeader struct { Timestamp string `json:"@timestamp,omitempty"` } +// PolicyOutput +type PolicyOutput struct { + ESDocument + + // API key the Elastic Agent uses to authenticate with elasticsearch + APIKey string `json:"api_key"` + + // Default API Key History + APIKeyHistory []APIKeyHistoryItems `json:"api_key_history"` + + // ID of the API key the Elastic Agent uses to authenticate with elasticsearch + APIKeyID string `json:"api_key_id"` + + // The policy output permissions hash + PolicyPermissionsHash string `json:"policy_permissions_hash"` +} + // Server A Fleet Server type Server struct { ESDocument diff --git a/model/schema.json b/model/schema.json index 09eda21a9..4e084b231 100644 --- a/model/schema.json +++ b/model/schema.json @@ -345,6 +345,50 @@ "server" ] }, + + + "policy_output" : { + "type": "object", + "properties": { + "api_key": { + "description": "API key the Elastic Agent uses to authenticate with elasticsearch", + "type": "string" + }, + "api_key_history": { + "description": "Default API Key History", + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "description": "API Key identifier", + "type": "string" + }, + "retired_at": { + "description": "Date/time the API key was retired", + "type": "string", + "format": "date-time" + } + } + } + }, + "api_key_id": { + "description": "ID of the API key the Elastic Agent uses to authenticate with elasticsearch", + "type": "string" + }, + "policy_permissions_hash": { + "description": "The policy output permissions hash", + "type": "string" + } + }, + "required": [ + "api_key", + "api_key_history", + "api_key_id", + "policy_permissions_hash" + ] + }, + "agent": { "title": "Agent", "description": "An Elastic Agent that has enrolled into Fleet", @@ -437,7 +481,7 @@ "type": "integer" }, "policy_output_permissions_hash": { - "description": "The policy output permissions hash", + "description": "Deprecated. Use Outputs instead. The policy output permissions hash", "type": "string" }, "last_updated": { @@ -455,15 +499,15 @@ "type": "string" }, "default_api_key_id": { - "description": "ID of the API key the Elastic Agent uses to authenticate with elasticsearch", + "description": "Deprecated. Use Outputs instead. ID of the API key the Elastic Agent uses to authenticate with elasticsearch", "type": "string" }, "default_api_key": { - "description": "API key the Elastic Agent uses to authenticate with elasticsearch", + "description": "Deprecated. Use Outputs instead. API key the Elastic Agent uses to authenticate with elasticsearch", "type": "string" }, "default_api_key_history": { - "description": "Default API Key History", + "description": "Deprecated. Use Outputs instead. Default API Key History", "type": "array", "items": { "type": "object", @@ -480,6 +524,11 @@ } } }, + "elasticsearch_outputs": { + "description": "ElasticsearchOutputs is the policy output data for each Elasticsearch output. It maps the output name to its data", + "type": "object", + "additionalProperties": { "$ref": "#/definitions/policy_output"} + }, "updated_at": { "description": "Date/time the Elastic Agent was last updated", "type": "string", From e31132187dcdf3b9aca44086d36f1116d9ffa91d Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 21 Jul 2022 11:08:11 +0200 Subject: [PATCH 32/89] no real change, tests are still passing --- internal/pkg/policy/parsed_policy.go | 2 +- internal/pkg/policy/parsed_policy_test.go | 1 - internal/pkg/policy/policy_output.go | 225 ++++++++++++---------- internal/pkg/policy/policy_output_test.go | 2 +- 4 files changed, 121 insertions(+), 109 deletions(-) diff --git a/internal/pkg/policy/parsed_policy.go b/internal/pkg/policy/parsed_policy.go index dbf5d3801..c8e2c7db0 100644 --- a/internal/pkg/policy/parsed_policy.go +++ b/internal/pkg/policy/parsed_policy.go @@ -126,13 +126,13 @@ func parsePerms(permsRaw json.RawMessage) (RoleMapT, error) { // iterate across the keys m := make(RoleMapT, len(permMap)) for k := range permMap { - v := permMap.GetMap(k) if v != nil { var r RoleT // Stable hash on permissions payload + // permission hash created here if r.Sha2, err = v.Hash(); err != nil { return nil, err } diff --git a/internal/pkg/policy/parsed_policy_test.go b/internal/pkg/policy/parsed_policy_test.go index 547cfcf7a..32ef271a7 100644 --- a/internal/pkg/policy/parsed_policy_test.go +++ b/internal/pkg/policy/parsed_policy_test.go @@ -13,7 +13,6 @@ import ( ) func TestNewParsedPolicy(t *testing.T) { - // Run two formatting of the same payload to validate that the sha2 remains the same payloads := []string{ testPolicy, diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index b172bd73b..e59764b26 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -38,121 +38,132 @@ type PolicyOutput struct { Role *RoleT } +// Prepare prepares the output p to be sent to the elastic-agent +// The agent might be mutated for an elasticsearch output func (p *PolicyOutput) Prepare(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, agent *model.Agent, outputMap smap.Map) error { switch p.Type { case OutputTypeElasticsearch: - zlog = zlog.With(). - Str("fleet.agent.id", agent.Id). - Str("fleet.policy.output.name", p.Name).Logger() - - zlog.Info().Msg("preparing elasticsearch output") - - // The role is required to do api key management - if p.Role == nil { - zlog.Error(). - Str("fleet.output.name", p.Name). - Msg("policy does not contain required output permission section") - return ErrNoOutputPerms + if err := p.prepareElasticsearch(ctx, zlog, bulker, agent, outputMap); err != nil { + return fmt.Errorf("failed to prepare elasticsearch output %q: %w", p.Name, err) } + case OutputTypeLogstash: + zlog.Debug().Msg("preparing logstash output") + zlog.Info().Msg("no actions required for logstash output preparation") + default: + zlog.Error().Msgf("unknown output type: %s; skipping preparation", p.Type) + return fmt.Errorf("encountered unexpected output type while preparing outputs: %s", p.Type) + } + return nil +} - // Determine whether we need to generate an output ApiKey. - // This is accomplished by comparing the sha2 hash stored in the agent - // record with the precalculated sha2 hash of the role. - - // Note: This will need to be updated when doing multi-cluster elasticsearch support - // Currently, we only have access to the token for the elasticsearch instance fleet-server - // is monitors. When updating for multiple ES instances we need to tie the token to the output. - needNewKey := true - switch { - case agent.DefaultAPIKey == "": - zlog.Info().Msg("must generate api key as default API key is not present") - case p.Role.Sha2 != agent.PolicyOutputPermissionsHash: - zlog.Info().Msg("must generate api key as policy output permissions changed") - default: - needNewKey = false - zlog.Info().Msg("policy output permissions are the same") +func (p *PolicyOutput) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, agent *model.Agent, outputMap smap.Map) error { + zlog = zlog.With(). + Str("fleet.agent.id", agent.Id). + Str("fleet.policy.output.name", p.Name).Logger() + + zlog.Info().Msg("preparing elasticsearch output") + + // The role is required to do api key management + if p.Role == nil { + zlog.Error(). + Str("fleet.output.name", p.Name). + Msg("policy does not contain required output permission section") + return ErrNoOutputPerms + } + + // Determine whether we need to generate an output ApiKey. + // This is accomplished by comparing the sha2 hash stored in the agent + // record with the precalculated sha2 hash of the role. + + // Note: This will need to be updated when doing multi-cluster elasticsearch support + // Currently, we only have access to the token for the elasticsearch instance fleet-server + // is monitors. When updating for multiple ES instances we need to tie the token to the output. + needNewKey := true + switch { + case agent.DefaultAPIKey == "": + zlog.Debug().Msg("must generate api key as default API key is not present") + case p.Role.Sha2 != agent.PolicyOutputPermissionsHash: + // the is actually the OutputPermissionsHash for the default hash. The Agent + // document on ES does not have OutputPermissionsHash for any other output + // besides the default one. It seems to me error-prone to rely on the default + // output permissions hash to generate new API keys for other outputs. + zlog.Debug().Msg("must generate api key as policy output permissions changed") + default: + needNewKey = false + zlog.Debug().Msg("policy output permissions are the same") + } + + if needNewKey { + zlog.Debug(). + RawJSON("fleet.policy.roles", p.Role.Raw). + Str("fleet.policy.default.oldHash", agent.PolicyOutputPermissionsHash). + Str("fleet.policy.default.newHash", p.Role.Sha2). + Msg("Generating a new API key") + + ctx := zlog.WithContext(ctx) + outputAPIKey, err := generateOutputAPIKey(ctx, bulker, agent.Id, p.Name, p.Role.Raw) + if err != nil { + zlog.Error().Err(err).Msg("fail generate output key") + return err } - if needNewKey { - zlog.Info(). - RawJSON("roles", p.Role.Raw). - Str("oldHash", agent.PolicyOutputPermissionsHash). - Str("newHash", p.Role.Sha2). - Msg("Generating a new API key") - - ctx := zlog.WithContext(ctx) - outputAPIKey, err := generateOutputAPIKey(ctx, bulker, agent.Id, p.Name, p.Role.Raw) - if err != nil { - zlog.Error().Err(err).Msg("fail generate output key") - return err - } - zlog.Info(). - Str("fleet.anderson.default.old.apikey", agent.DefaultAPIKey). - Str("fleet.anderson.default.new.apikey", outputAPIKey.Agent()). - Msg("setting / swapping agent default API key") - agent.DefaultAPIKey = outputAPIKey.Agent() - - // When a new keys is generated we need to update the Agent record, - // this will need to be updated when multiples Elasticsearch output - // are used. - zlog.Info(). - Str("fleet.role.hash.sha256", p.Role.Sha2). - Str(logger.DefaultOutputAPIKeyID, outputAPIKey.ID). - Msg("Updating agent record to pick up default output key.") - - fields := map[string]interface{}{ - dl.FieldDefaultAPIKey: outputAPIKey.Agent(), - dl.FieldDefaultAPIKeyID: outputAPIKey.ID, - dl.FieldPolicyOutputPermissionsHash: p.Role.Sha2, - } - if agent.DefaultAPIKeyID != "" { - fields[dl.FieldDefaultAPIKeyHistory] = model.DefaultAPIKeyHistoryItems{ - ID: agent.DefaultAPIKeyID, - RetiredAt: time.Now().UTC().Format(time.RFC3339), - } - } + agent.DefaultAPIKey = outputAPIKey.Agent() - // Using painless script to append the old keys to the history - body, err := renderUpdatePainlessScript(fields) - if err != nil { - return fmt.Errorf("could no tupdate painless script: %w", err) - } + // When a new keys is generated we need to update the Agent record, + // this will need to be updated when multiples Elasticsearch output + // are used. + zlog.Info(). + Str("fleet.role.hash.sha256", p.Role.Sha2). + Str(logger.DefaultOutputAPIKeyID, outputAPIKey.ID). + Msg("Updating agent record to pick up default output key.") - if err = bulker.Update(ctx, dl.FleetAgents, agent.Id, body); err != nil { - zlog.Error().Err(err).Msg("fail update agent record") - return fmt.Errorf("fail update agent record: %w", err) + fields := map[string]interface{}{ + dl.FieldDefaultAPIKey: outputAPIKey.Agent(), + dl.FieldDefaultAPIKeyID: outputAPIKey.ID, + dl.FieldPolicyOutputPermissionsHash: p.Role.Sha2, + } + if agent.DefaultAPIKeyID != "" { + fields[dl.FieldDefaultAPIKeyHistory] = model.DefaultAPIKeyHistoryItems{ + ID: agent.DefaultAPIKeyID, + RetiredAt: time.Now().UTC().Format(time.RFC3339), } } - // Always insert the `api_key` as part of the output block, this is required - // because only fleet server knows the api key for the specific agent, if we don't - // add it the agent will not receive the `api_key` and will not be able to connect - // to Elasticsearch. - // - // We need to investigate allocation with the new LS output, we had optimization - // in place to reduce number of agent policy allocation when sending the updated - // agent policy to multiple agents. - // See: https://github.com/elastic/fleet-server/issues/1301 - // - // WIP: - // The agent struct is a pointer, thus shared between the runs of Prepare - // for each output. Given 2 outputs (1 and 2), if 1 gets a new API key, - // agent.DefaultAPIKey changes. However, if 2 does not get one, the code - // below runs anyway, therefore the API key for output 2 will be set to the - // api key for output 1 - // The agent struct cannot be shared! The new API key must be its own local - // variable and a special case for the 'default output' has to handle - // updating agent.DefaultAPIKey. - if ok := setMapObj(outputMap, agent.DefaultAPIKey, p.Name, "api_key"); !ok { - return ErrFailInjectAPIKey + // Using painless script to append the old keys to the history + body, err := renderUpdatePainlessScript(fields) + if err != nil { + return fmt.Errorf("could no tupdate painless script: %w", err) + } + + if err = bulker.Update(ctx, dl.FleetAgents, agent.Id, body); err != nil { + zlog.Error().Err(err).Msg("fail update agent record") + return fmt.Errorf("fail update agent record: %w", err) } - case OutputTypeLogstash: - zlog.Info().Msg("preparing logstash output") - zlog.Info().Msg("no actions required for logstash output preparation") - default: - zlog.Error().Msgf("unknown output type: %s; skipping preparation", p.Type) - return fmt.Errorf("encountered unexpected output type while preparing outputs: %s", p.Type) } + + // Always insert the `api_key` as part of the output block, this is required + // because only fleet server knows the api key for the specific agent, if we don't + // add it the agent will not receive the `api_key` and will not be able to connect + // to Elasticsearch. + // + // We need to investigate allocation with the new LS output, we had optimization + // in place to reduce number of agent policy allocation when sending the updated + // agent policy to multiple agents. + // See: https://github.com/elastic/fleet-server/issues/1301 + // + // WIP: + // The agent struct is a pointer, thus shared between the runs of Prepare + // for each output. Given 2 outputs (1 and 2), if 1 gets a new API key, + // agent.DefaultAPIKey changes. However, if 2 does not get one, the code + // below runs anyway, therefore the API key for output 2 will be set to the + // api key for output 1 + // The agent struct cannot be shared! The new API key must be its own local + // variable and a special case for the 'default output' has to handle + // updating agent.DefaultAPIKey. + if err := setMapObj(outputMap, agent.DefaultAPIKey, p.Name, "api_key"); err != nil { + return err + } + return nil } @@ -190,25 +201,27 @@ func generateOutputAPIKey(ctx context.Context, bulk bulk.Bulk, agentID, outputNa ) } -func setMapObj(obj map[string]interface{}, val interface{}, keys ...string) bool { +func setMapObj(obj map[string]interface{}, val interface{}, keys ...string) error { if len(keys) == 0 { - return false + return fmt.Errorf("no key to be updated: %w", ErrFailInjectAPIKey) } for _, k := range keys[:len(keys)-1] { v, ok := obj[k] if !ok { - return false + return fmt.Errorf("no key %q not present on MapObj: %w", + k, ErrFailInjectAPIKey) } obj, ok = v.(map[string]interface{}) if !ok { - return false + return fmt.Errorf("cannot cast %T to map[string]interface{}: %w", + obj, ErrFailInjectAPIKey) } } k := keys[len(keys)-1] obj[k] = val - return true + return nil } diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go index 1e90cee57..861fdafa3 100644 --- a/internal/pkg/policy/policy_output_test.go +++ b/internal/pkg/policy/policy_output_test.go @@ -155,7 +155,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { logger := testlog.SetLogger(t) bulker := ftesting.NewMockBulk() bulker.On("Update", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() - bulker.On("APIKeyCreate", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&bulk.APIKey{"abc", "new-key"}, nil).Once() //nolint:govet // test case + bulker.On("APIKeyCreate", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&bulk.APIKey{ID: "abc", Key: "new-key"}, nil).Once() //nolint:govet // test case po := PolicyOutput{ Type: OutputTypeElasticsearch, From 68e6bf20f50459949b4f0e5de9fe9d35e59315c6 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 21 Jul 2022 11:08:17 +0200 Subject: [PATCH 33/89] notes --- 0.notes.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/0.notes.md b/0.notes.md index 36c955a25..a106401e2 100644 --- a/0.notes.md +++ b/0.notes.md @@ -1,3 +1,13 @@ +1. Add fields to the .agent fields, don't change the existing ones +2. on upgrade bump the coordinator index +3. on upgrade set the default api to empty -> force all API keys to be regenerated +4. on agent checkin, after upgrade, try to search api key by metadata.agentId and invalidate the "old" ones + + + + + + Thu 14 Jul 16:09:41 CEST 2022 so far I could drill the problem down to how the output permissions change. From 58a7eab830a5c7f5f6d228ebebab68481a23ea26 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 21 Jul 2022 11:09:01 +0200 Subject: [PATCH 34/89] no real change --- internal/pkg/api/handleCheckin.go | 12 +++++++----- internal/pkg/coordinator/monitor.go | 2 ++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/pkg/api/handleCheckin.go b/internal/pkg/api/handleCheckin.go index f95c0769f..2d38cb67b 100644 --- a/internal/pkg/api/handleCheckin.go +++ b/internal/pkg/api/handleCheckin.go @@ -10,6 +10,7 @@ import ( "compress/gzip" "context" "encoding/json" + "fmt" "math/rand" "net/http" "reflect" @@ -430,13 +431,13 @@ func convertActions(agentID string, actions []model.Action) ([]ActionResp, strin // func processPolicy(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, agentID string, pp *policy.ParsedPolicy) (*ActionResp, error) { zlog = zlog.With(). - Str("ctx", "processPolicy"). - Int64("policyRevision", pp.Policy.RevisionIdx). - Int64("policyCoordinator", pp.Policy.CoordinatorIdx). + Str("fleet.ctx", "processPolicy"). + Int64("fleet.policyRevision", pp.Policy.RevisionIdx). + Int64("fleet.policyCoordinator", pp.Policy.CoordinatorIdx). Str(LogPolicyID, pp.Policy.PolicyID). Logger() - // Repull and decode the agent object. Do not trust the cache. + // Repull and decode the agent object. Do not trust the cache. agent, err := dl.FindAgent(ctx, bulker, dl.QueryAgentByID, dl.FieldID, agentID) if err != nil { zlog.Error().Err(err).Msg("fail find agent record") @@ -458,7 +459,8 @@ func processPolicy(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, a for _, policyOutput := range pp.Outputs { err = policyOutput.Prepare(ctx, zlog, bulker, &agent, outputs) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to prepare output %q: %w", + policyOutput.Name, err) } } diff --git a/internal/pkg/coordinator/monitor.go b/internal/pkg/coordinator/monitor.go index e43f5b44a..7be4904fc 100644 --- a/internal/pkg/coordinator/monitor.go +++ b/internal/pkg/coordinator/monitor.go @@ -545,5 +545,7 @@ func getAPIKeyIDs(agent *model.Agent) []string { if agent.DefaultAPIKeyID != "" { keys = append(keys, agent.DefaultAPIKeyID) } + // TODO: should we also collect the old (a.k.a history) api keys to ensure + // they're deleted? return keys } From 495ea7d3fb728267619ae8b843a30062b26427b7 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 21 Jul 2022 11:25:32 +0200 Subject: [PATCH 35/89] naming convention --- internal/pkg/policy/parsed_policy.go | 8 ++++---- internal/pkg/policy/policy_output.go | 6 +++--- internal/pkg/policy/policy_output_test.go | 14 +++++++------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/internal/pkg/policy/parsed_policy.go b/internal/pkg/policy/parsed_policy.go index c8e2c7db0..029298ef5 100644 --- a/internal/pkg/policy/parsed_policy.go +++ b/internal/pkg/policy/parsed_policy.go @@ -42,7 +42,7 @@ type ParsedPolicy struct { Policy model.Policy Fields map[string]json.RawMessage Roles RoleMapT - Outputs map[string]PolicyOutput + Outputs map[string]Output Default ParsedPolicyDefaults } @@ -91,8 +91,8 @@ func NewParsedPolicy(p model.Policy) (*ParsedPolicy, error) { return pp, nil } -func constructPolicyOutputs(outputsRaw json.RawMessage, roles map[string]RoleT) (map[string]PolicyOutput, error) { - result := make(map[string]PolicyOutput) +func constructPolicyOutputs(outputsRaw json.RawMessage, roles map[string]RoleT) (map[string]Output, error) { + result := make(map[string]Output) outputsMap, err := smap.Parse(outputsRaw) if err != nil { @@ -102,7 +102,7 @@ func constructPolicyOutputs(outputsRaw json.RawMessage, roles map[string]RoleT) for k := range outputsMap { v := outputsMap.GetMap(k) - p := PolicyOutput{ + p := Output{ Name: k, Type: v.GetString(FieldOutputType), } diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index e59764b26..4cf798c03 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -32,7 +32,7 @@ var ( ErrFailInjectAPIKey = errors.New("fail inject api key") ) -type PolicyOutput struct { +type Output struct { Name string Type string Role *RoleT @@ -40,7 +40,7 @@ type PolicyOutput struct { // Prepare prepares the output p to be sent to the elastic-agent // The agent might be mutated for an elasticsearch output -func (p *PolicyOutput) Prepare(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, agent *model.Agent, outputMap smap.Map) error { +func (p *Output) Prepare(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, agent *model.Agent, outputMap smap.Map) error { switch p.Type { case OutputTypeElasticsearch: if err := p.prepareElasticsearch(ctx, zlog, bulker, agent, outputMap); err != nil { @@ -56,7 +56,7 @@ func (p *PolicyOutput) Prepare(ctx context.Context, zlog zerolog.Logger, bulker return nil } -func (p *PolicyOutput) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, agent *model.Agent, outputMap smap.Map) error { +func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, agent *model.Agent, outputMap smap.Map) error { zlog = zlog.With(). Str("fleet.agent.id", agent.Id). Str("fleet.policy.output.name", p.Name).Logger() diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go index 861fdafa3..676e5c363 100644 --- a/internal/pkg/policy/policy_output_test.go +++ b/internal/pkg/policy/policy_output_test.go @@ -23,7 +23,7 @@ var TestPayload []byte func TestPolicyLogstashOutputPrepare(t *testing.T) { logger := testlog.SetLogger(t) bulker := ftesting.NewMockBulk() - po := PolicyOutput{ + po := Output{ Type: OutputTypeLogstash, Name: "test output", Role: &RoleT{ @@ -39,7 +39,7 @@ func TestPolicyLogstashOutputPrepare(t *testing.T) { func TestPolicyLogstashOutputPrepareNoRole(t *testing.T) { logger := testlog.SetLogger(t) bulker := ftesting.NewMockBulk() - po := PolicyOutput{ + po := Output{ Type: OutputTypeLogstash, Name: "test output", Role: nil, @@ -54,7 +54,7 @@ func TestPolicyLogstashOutputPrepareNoRole(t *testing.T) { func TestPolicyDefaultLogstashOutputPrepare(t *testing.T) { logger := testlog.SetLogger(t) bulker := ftesting.NewMockBulk() - po := PolicyOutput{ + po := Output{ Type: OutputTypeLogstash, Name: "test output", Role: &RoleT{ @@ -71,7 +71,7 @@ func TestPolicyDefaultLogstashOutputPrepare(t *testing.T) { func TestPolicyESOutputPrepareNoRole(t *testing.T) { logger := testlog.SetLogger(t) bulker := ftesting.NewMockBulk() - po := PolicyOutput{ + po := Output{ Type: OutputTypeElasticsearch, Name: "test output", Role: nil, @@ -87,7 +87,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { logger := testlog.SetLogger(t) bulker := ftesting.NewMockBulk() hashPerm := "abc123" - po := PolicyOutput{ + po := Output{ Type: OutputTypeElasticsearch, Name: "test output", Role: &RoleT{ @@ -123,7 +123,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { bulker.On("Update", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() bulker.On("APIKeyCreate", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&bulk.APIKey{"abc", "new-key"}, nil).Once() //nolint:govet // test case - po := PolicyOutput{ + po := Output{ Type: OutputTypeElasticsearch, Name: "test output", Role: &RoleT{ @@ -157,7 +157,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { bulker.On("Update", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() bulker.On("APIKeyCreate", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&bulk.APIKey{ID: "abc", Key: "new-key"}, nil).Once() //nolint:govet // test case - po := PolicyOutput{ + po := Output{ Type: OutputTypeElasticsearch, Name: "test output", Role: &RoleT{ From 52b8aa0e77adf81d4a7b93a7163c309562378cc6 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 21 Jul 2022 11:25:49 +0200 Subject: [PATCH 36/89] TODO --- internal/pkg/policy/policy_output.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index 4cf798c03..6f2a5661c 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -71,6 +71,10 @@ func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, return ErrNoOutputPerms } + // 1 - just use the new ElasticsearchOutputs + // 2 - make the tests check if they're correctly filled in + // 3 - ensure Default* is made empty/nil at the end + // Determine whether we need to generate an output ApiKey. // This is accomplished by comparing the sha2 hash stored in the agent // record with the precalculated sha2 hash of the role. From 35c05f0148a9712420084d6c469c5f1dba7dfa12 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 21 Jul 2022 12:07:51 +0200 Subject: [PATCH 37/89] adjust DefaultAPIKeyHistoryItems definition --- internal/pkg/model/schema.go | 12 +-------- model/schema.json | 50 +++++++++++++++--------------------- 2 files changed, 21 insertions(+), 41 deletions(-) diff --git a/internal/pkg/model/schema.go b/internal/pkg/model/schema.go index 086b044f1..a82d434d2 100644 --- a/internal/pkg/model/schema.go +++ b/internal/pkg/model/schema.go @@ -134,7 +134,7 @@ type Agent struct { DefaultAPIKey string `json:"default_api_key,omitempty"` // Deprecated. Use Outputs instead. Default API Key History - DefaultAPIKeyHistory []DefaultAPIKeyHistoryItems `json:"default_api_key_history,omitempty"` + DefaultAPIKeyHistory []APIKeyHistoryItems `json:"default_api_key_history,omitempty"` // Deprecated. Use Outputs instead. ID of the API key the Elastic Agent uses to authenticate with elasticsearch DefaultAPIKeyID string `json:"default_api_key_id,omitempty"` @@ -256,16 +256,6 @@ type Body struct { type Data struct { } -// DefaultAPIKeyHistoryItems -type DefaultAPIKeyHistoryItems struct { - - // API Key identifier - ID string `json:"id,omitempty"` - - // Date/time the API key was retired - RetiredAt string `json:"retired_at,omitempty"` -} - // EnrollmentAPIKey An Elastic Agent enrollment API key type EnrollmentAPIKey struct { ESDocument diff --git a/model/schema.json b/model/schema.json index 4e084b231..0df7bba75 100644 --- a/model/schema.json +++ b/model/schema.json @@ -346,6 +346,24 @@ ] }, + "api_key_history": { + "description": "Output API Key History", + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "description": "API Key identifier", + "type": "string" + }, + "retired_at": { + "description": "Date/time the API key was retired", + "type": "string", + "format": "date-time" + } + } + } + }, "policy_output" : { "type": "object", @@ -356,21 +374,7 @@ }, "api_key_history": { "description": "Default API Key History", - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "description": "API Key identifier", - "type": "string" - }, - "retired_at": { - "description": "Date/time the API key was retired", - "type": "string", - "format": "date-time" - } - } - } + "$ref": "#/definitions/api_key_history" }, "api_key_id": { "description": "ID of the API key the Elastic Agent uses to authenticate with elasticsearch", @@ -508,21 +512,7 @@ }, "default_api_key_history": { "description": "Deprecated. Use Outputs instead. Default API Key History", - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "description": "API Key identifier", - "type": "string" - }, - "retired_at": { - "description": "Date/time the API key was retired", - "type": "string", - "format": "date-time" - } - } - } + "$ref": "#/definitions/api_key_history" }, "elasticsearch_outputs": { "description": "ElasticsearchOutputs is the policy output data for each Elasticsearch output. It maps the output name to its data", From a7e32176ae0570eab724fbb39fe9977d93501f69 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 21 Jul 2022 12:08:20 +0200 Subject: [PATCH 38/89] adjust Prepare to use new model.Agent --- internal/pkg/api/handleCheckin.go | 4 + internal/pkg/policy/policy_output.go | 36 +++++++-- internal/pkg/policy/policy_output_test.go | 89 +++++++++++++++++------ 3 files changed, 97 insertions(+), 32 deletions(-) diff --git a/internal/pkg/api/handleCheckin.go b/internal/pkg/api/handleCheckin.go index 2d38cb67b..a93c25407 100644 --- a/internal/pkg/api/handleCheckin.go +++ b/internal/pkg/api/handleCheckin.go @@ -443,6 +443,10 @@ func processPolicy(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, a zlog.Error().Err(err).Msg("fail find agent record") return nil, err } + // TODO: do we need to kee it here? + if agent.ElasticsearchOutputs == nil { + agent.ElasticsearchOutputs = map[string]*model.PolicyOutput{} + } // Parse the outputs maps in order to prepare the outputs const outputsProperty = "outputs" diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index 6f2a5661c..b22895a11 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -75,6 +75,26 @@ func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, // 2 - make the tests check if they're correctly filled in // 3 - ensure Default* is made empty/nil at the end + output, ok := agent.ElasticsearchOutputs[p.Name] + if !ok { + zlog.Debug().Msgf("creating agent.ElasticsearchOutputs[%s]", p.Name) + output = &model.PolicyOutput{} + agent.ElasticsearchOutputs[p.Name] = output + } + + // Migration path, use agent.ElasticsearchOutputs instead of agent.Default* + if agent.DefaultAPIKey != "" { + output.APIKey = agent.DefaultAPIKey + output.APIKeyID = agent.DefaultAPIKeyID + output.PolicyPermissionsHash = agent.PolicyOutputPermissionsHash + output.APIKeyHistory = agent.DefaultAPIKeyHistory + + agent.DefaultAPIKey = "" + agent.DefaultAPIKeyID = "" + agent.PolicyOutputPermissionsHash = "" + agent.DefaultAPIKeyHistory = nil + } + // Determine whether we need to generate an output ApiKey. // This is accomplished by comparing the sha2 hash stored in the agent // record with the precalculated sha2 hash of the role. @@ -84,9 +104,9 @@ func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, // is monitors. When updating for multiple ES instances we need to tie the token to the output. needNewKey := true switch { - case agent.DefaultAPIKey == "": + case output.APIKey == "": zlog.Debug().Msg("must generate api key as default API key is not present") - case p.Role.Sha2 != agent.PolicyOutputPermissionsHash: + case p.Role.Sha2 != output.PolicyPermissionsHash: // the is actually the OutputPermissionsHash for the default hash. The Agent // document on ES does not have OutputPermissionsHash for any other output // besides the default one. It seems to me error-prone to rely on the default @@ -100,7 +120,7 @@ func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, if needNewKey { zlog.Debug(). RawJSON("fleet.policy.roles", p.Role.Raw). - Str("fleet.policy.default.oldHash", agent.PolicyOutputPermissionsHash). + Str("fleet.policy.default.oldHash", output.PolicyPermissionsHash). Str("fleet.policy.default.newHash", p.Role.Sha2). Msg("Generating a new API key") @@ -111,7 +131,7 @@ func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, return err } - agent.DefaultAPIKey = outputAPIKey.Agent() + output.APIKey = outputAPIKey.Agent() // When a new keys is generated we need to update the Agent record, // this will need to be updated when multiples Elasticsearch output @@ -126,9 +146,9 @@ func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, dl.FieldDefaultAPIKeyID: outputAPIKey.ID, dl.FieldPolicyOutputPermissionsHash: p.Role.Sha2, } - if agent.DefaultAPIKeyID != "" { - fields[dl.FieldDefaultAPIKeyHistory] = model.DefaultAPIKeyHistoryItems{ - ID: agent.DefaultAPIKeyID, + if agent.DefaultAPIKeyID != "" { // TODO: output.APIKeyID != "" + fields[dl.FieldDefaultAPIKeyHistory] = model.APIKeyHistoryItems{ + ID: output.APIKeyID, RetiredAt: time.Now().UTC().Format(time.RFC3339), } } @@ -164,7 +184,7 @@ func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, // The agent struct cannot be shared! The new API key must be its own local // variable and a special case for the 'default output' has to handle // updating agent.DefaultAPIKey. - if err := setMapObj(outputMap, agent.DefaultAPIKey, p.Name, "api_key"); err != nil { + if err := setMapObj(outputMap, output.APIKey, p.Name, "api_key"); err != nil { return err } diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go index 676e5c363..366737fba 100644 --- a/internal/pkg/policy/policy_output_test.go +++ b/internal/pkg/policy/policy_output_test.go @@ -8,6 +8,7 @@ import ( "context" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -86,8 +87,9 @@ func TestPolicyOutputESPrepare(t *testing.T) { t.Run("Permission hash == Agent Permission Hash no need to regenerate the key", func(t *testing.T) { logger := testlog.SetLogger(t) bulker := ftesting.NewMockBulk() + apiKey := bulk.APIKey{ID: "test_id", Key: "EXISTING-KEY"} hashPerm := "abc123" - po := Output{ + output := Output{ Type: OutputTypeElasticsearch, Name: "test output", Role: &RoleT{ @@ -101,29 +103,51 @@ func TestPolicyOutputESPrepare(t *testing.T) { } testAgent := &model.Agent{ - DefaultAPIKey: "test_id:EXISTING-KEY", + DefaultAPIKey: apiKey.Agent(), PolicyOutputPermissionsHash: hashPerm, + ElasticsearchOutputs: map[string]*model.PolicyOutput{ + output.Name: { + ESDocument: model.ESDocument{}, + APIKey: apiKey.Agent(), + APIKeyHistory: nil, + APIKeyID: apiKey.ID, + PolicyPermissionsHash: hashPerm, + }, + }, } - err := po.Prepare(context.Background(), logger, bulker, testAgent, policyMap) + err := output.Prepare(context.Background(), logger, bulker, testAgent, policyMap) require.NoError(t, err, "expected prepare to pass") - key, ok := policyMap.GetMap("test output")["api_key"].(string) + key, ok := policyMap.GetMap(output.Name)["api_key"].(string) - require.True(t, ok, "unable to case api key") - require.Equal(t, testAgent.DefaultAPIKey, key) - bulker.AssertNotCalled(t, "Update", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) - bulker.AssertNotCalled(t, "APIKeyCreate", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) + assert.True(t, ok, "api key not present on policy map") + assert.Equal(t, testAgent.ElasticsearchOutputs[output.Name].APIKey, key) + assert.Empty(t, testAgent.DefaultAPIKey) // Migration path: ensure we don't use DefaultAPIKey anymore + + bulker.AssertNotCalled(t, "Update", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) + bulker.AssertNotCalled(t, "APIKeyCreate", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) bulker.AssertExpectations(t) }) t.Run("Permission hash != Agent Permission Hash need to regenerate the key", func(t *testing.T) { logger := testlog.SetLogger(t) bulker := ftesting.NewMockBulk() - bulker.On("Update", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() - bulker.On("APIKeyCreate", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&bulk.APIKey{"abc", "new-key"}, nil).Once() //nolint:govet // test case - po := Output{ + oldAPIKey := bulk.APIKey{ID: "test_id", Key: "EXISTING-KEY"} + wantAPIKey := bulk.APIKey{ID: "abc", Key: "new-key"} + hashPerm := "old-HASH" + + bulker.On("Update", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(nil).Once() + bulker.On("APIKeyCreate", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(&wantAPIKey, nil).Once() //nolint:govet // test case + + output := Output{ Type: OutputTypeElasticsearch, Name: "test output", Role: &RoleT{ @@ -137,27 +161,44 @@ func TestPolicyOutputESPrepare(t *testing.T) { } testAgent := &model.Agent{ - DefaultAPIKey: "test_id:EXISTING-KEY", - PolicyOutputPermissionsHash: "old-HASH", + DefaultAPIKey: oldAPIKey.Agent(), + PolicyOutputPermissionsHash: hashPerm, + ElasticsearchOutputs: map[string]*model.PolicyOutput{ + output.Name: { + ESDocument: model.ESDocument{}, + APIKey: oldAPIKey.Agent(), + APIKeyHistory: nil, + APIKeyID: oldAPIKey.ID, + PolicyPermissionsHash: hashPerm, + }, + }, } - err := po.Prepare(context.Background(), logger, bulker, testAgent, policyMap) + err := output.Prepare(context.Background(), logger, bulker, testAgent, policyMap) require.NoError(t, err, "expected prepare to pass") - key, ok := policyMap.GetMap("test output")["api_key"].(string) + key, ok := policyMap.GetMap(output.Name)["api_key"].(string) require.True(t, ok, "unable to case api key") - require.Equal(t, "abc:new-key", key) + require.Equal(t, wantAPIKey.Agent(), key) + assert.Empty(t, testAgent.DefaultAPIKey) // Migration path: ensure we don't use DefaultAPIKey anymore + assert.Equal(t, testAgent.ElasticsearchOutputs[output.Name].APIKey, key) + bulker.AssertExpectations(t) }) t.Run("Generate API Key on new Agent", func(t *testing.T) { logger := testlog.SetLogger(t) bulker := ftesting.NewMockBulk() - bulker.On("Update", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() - bulker.On("APIKeyCreate", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&bulk.APIKey{ID: "abc", Key: "new-key"}, nil).Once() //nolint:govet // test case - - po := Output{ + bulker.On("Update", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(nil).Once() + apiKey := bulk.APIKey{ID: "abc", Key: "new-key"} + bulker.On("APIKeyCreate", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(&apiKey, nil).Once() //nolint:govet // test case + + output := Output{ Type: OutputTypeElasticsearch, Name: "test output", Role: &RoleT{ @@ -170,15 +211,15 @@ func TestPolicyOutputESPrepare(t *testing.T) { "test output": map[string]interface{}{}, } - testAgent := &model.Agent{} + testAgent := &model.Agent{ElasticsearchOutputs: map[string]*model.PolicyOutput{}} - err := po.Prepare(context.Background(), logger, bulker, testAgent, policyMap) + err := output.Prepare(context.Background(), logger, bulker, testAgent, policyMap) require.NoError(t, err, "expected prepare to pass") - key, ok := policyMap.GetMap("test output")["api_key"].(string) + key, ok := policyMap.GetMap(output.Name)["api_key"].(string) require.True(t, ok, "unable to case api key") - require.Equal(t, "abc:new-key", key) + require.Equal(t, apiKey.Agent(), key) bulker.AssertExpectations(t) }) } From be825967bb27531cb1f3808fbdc75a2fe06f228d Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 21 Jul 2022 16:12:43 +0200 Subject: [PATCH 39/89] adjust find Agent functions --- internal/pkg/coordinator/monitor.go | 2 +- internal/pkg/dl/agent.go | 28 ++++++--- internal/pkg/dl/agent_integration_test.go | 72 +++++++++++++++++++++++ 3 files changed, 93 insertions(+), 9 deletions(-) diff --git a/internal/pkg/coordinator/monitor.go b/internal/pkg/coordinator/monitor.go index 7be4904fc..39d99abea 100644 --- a/internal/pkg/coordinator/monitor.go +++ b/internal/pkg/coordinator/monitor.go @@ -478,7 +478,7 @@ func runUnenroller(ctx context.Context, bulker bulk.Bulk, policyID string, unenr func runUnenrollerWork(ctx context.Context, bulker bulk.Bulk, policyID string, unenrollTimeout time.Duration, zlog zerolog.Logger, agentsIndex string) error { agents, err := dl.FindOfflineAgents(ctx, bulker, policyID, unenrollTimeout, dl.WithIndexName(agentsIndex)) - if err != nil || len(agents) == 0 { + if err != nil { return err } diff --git a/internal/pkg/dl/agent.go b/internal/pkg/dl/agent.go index 1d52082f7..5b82cdb9d 100644 --- a/internal/pkg/dl/agent.go +++ b/internal/pkg/dl/agent.go @@ -6,6 +6,7 @@ package dl import ( "context" + "fmt" "time" "github.com/elastic/fleet-server/v7/internal/pkg/bulk" @@ -48,19 +49,26 @@ func prepareOfflineAgentsByPolicyID() *dsl.Tmpl { return tmpl } -func FindAgent(ctx context.Context, bulker bulk.Bulk, tmpl *dsl.Tmpl, name string, v interface{}, opt ...Option) (agent model.Agent, err error) { +func FindAgent(ctx context.Context, bulker bulk.Bulk, tmpl *dsl.Tmpl, name string, v interface{}, opt ...Option) (model.Agent, error) { o := newOption(FleetAgents, opt...) res, err := SearchWithOneParam(ctx, bulker, tmpl, o.indexName, name, v) if err != nil { - return + return model.Agent{}, fmt.Errorf("failed searching for agent: %w", err) } if len(res.Hits) == 0 { - return agent, ErrNotFound + return model.Agent{}, ErrNotFound } - err = res.Hits[0].Unmarshal(&agent) - return agent, err + var agent model.Agent + if err = res.Hits[0].Unmarshal(&agent); err != nil { + return model.Agent{}, fmt.Errorf("could not unmarshal ES document into model.Agent: %w", err) + } + + if agent.ElasticsearchOutputs == nil { + agent.ElasticsearchOutputs = map[string]*model.PolicyOutput{} + } + return agent, nil } func FindOfflineAgents(ctx context.Context, bulker bulk.Bulk, policyID string, unenrollTimeout time.Duration, opt ...Option) ([]model.Agent, error) { @@ -71,18 +79,22 @@ func FindOfflineAgents(ctx context.Context, bulker bulk.Bulk, policyID string, u FieldLastCheckin: past, }) if err != nil { - return nil, err + return nil, fmt.Errorf("failed searching for agent: %w", err) } if len(res.Hits) == 0 { - return nil, nil + return nil, ErrNotFound } agents := make([]model.Agent, len(res.Hits)) for i, hit := range res.Hits { if err := hit.Unmarshal(&agents[i]); err != nil { - return nil, err + return nil, fmt.Errorf("could not unmarshal ES document into model.Agent: %w", err) + } + if agents[i].ElasticsearchOutputs == nil { + agents[i].ElasticsearchOutputs = map[string]*model.PolicyOutput{} } } + return agents, nil } diff --git a/internal/pkg/dl/agent_integration_test.go b/internal/pkg/dl/agent_integration_test.go index 4e65ddb94..f519d7320 100644 --- a/internal/pkg/dl/agent_integration_test.go +++ b/internal/pkg/dl/agent_integration_test.go @@ -108,3 +108,75 @@ func TestFindOfflineAgents(t *testing.T) { require.Len(t, agents, 2) assert.EqualValues(t, []string{twoDayOldID, threeDayOldID}, []string{agents[0].Id, agents[1].Id}) } + +func TestFindAgent_NewModel(t *testing.T) { + index, bulker := ftesting.SetupCleanIndex(context.Background(), t, FleetAgents) + + now := time.Now().UTC() + nowStr := now.Format(time.RFC3339) + + policyID := uuid.Must(uuid.NewV4()).String() + agentID := uuid.Must(uuid.NewV4()).String() + + wantElasticsearchOutputs := map[string]*model.PolicyOutput{ + "default": { + APIKey: "TestFindNewModelAgent_APIKey", + APIKeyHistory: []model.APIKeyHistoryItems{ + { + ID: "TestFindNewModelAgent_APIKeyID_invalidated", + RetiredAt: "TestFindNewModelAgent_APIKeyID_invalidated_at"}, + }, + APIKeyID: "TestFindNewModelAgent_APIKeyID", + PolicyPermissionsHash: "TestFindNewModelAgent_PolicyPermissionsHash", + }, + } + body, err := json.Marshal(model.Agent{ + PolicyID: policyID, + Active: true, + LastCheckin: nowStr, + LastCheckinStatus: "", + UpdatedAt: nowStr, + EnrolledAt: nowStr, + ElasticsearchOutputs: wantElasticsearchOutputs, + }) + require.NoError(t, err) + + _, err = bulker.Create( + context.Background(), index, agentID, body, bulk.WithRefresh()) + require.NoError(t, err) + + agent, err := FindAgent( + context.Background(), bulker, QueryAgentByID, FieldID, agentID, WithIndexName(index)) + require.NoError(t, err) + + assert.Equal(t, agentID, agent.Id) + assert.Equal(t, wantElasticsearchOutputs, agent.ElasticsearchOutputs) +} + +func TestFindAgent_ESOutput_never_nil(t *testing.T) { + index, bulker := ftesting.SetupCleanIndex(context.Background(), t, FleetAgents) + + now := time.Now().UTC().Format(time.RFC3339) + + policyID := uuid.Must(uuid.NewV4()).String() + agentID := uuid.Must(uuid.NewV4()).String() + + body, err := json.Marshal(model.Agent{ + PolicyID: policyID, + Active: true, + LastCheckin: now, + LastCheckinStatus: "", + UpdatedAt: now, + EnrolledAt: now, + }) + require.NoError(t, err) + + _, err = bulker.Create(context.Background(), index, agentID, body, bulk.WithRefresh()) + require.NoError(t, err) + + agent, err := FindAgent(context.Background(), bulker, QueryAgentByID, FieldID, agentID, WithIndexName(index)) + require.NoError(t, err) + + assert.Equal(t, agentID, agent.Id) + assert.NotNil(t, agent.ElasticsearchOutputs) +} From 7b5e4aaf4332e3b433b7b860f172a35103c407a4 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 21 Jul 2022 16:13:12 +0200 Subject: [PATCH 40/89] do todo --- internal/pkg/api/handleCheckin.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/pkg/api/handleCheckin.go b/internal/pkg/api/handleCheckin.go index a93c25407..2d38cb67b 100644 --- a/internal/pkg/api/handleCheckin.go +++ b/internal/pkg/api/handleCheckin.go @@ -443,10 +443,6 @@ func processPolicy(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, a zlog.Error().Err(err).Msg("fail find agent record") return nil, err } - // TODO: do we need to kee it here? - if agent.ElasticsearchOutputs == nil { - agent.ElasticsearchOutputs = map[string]*model.PolicyOutput{} - } // Parse the outputs maps in order to prepare the outputs const outputsProperty = "outputs" From 40490c6490c77c1b0ff9516d0a5ad158d5ff5053 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Fri, 22 Jul 2022 17:58:52 +0200 Subject: [PATCH 41/89] wip --- 0.notes.md | 2 + internal/pkg/dl/agent_integration_test.go | 2 +- internal/pkg/model/schema.go | 30 ++-- internal/pkg/policy/policy_output.go | 27 ++- internal/pkg/policy/policy_output_test.go | 191 ++++++++++++++++++++-- model/schema.json | 13 +- 6 files changed, 221 insertions(+), 44 deletions(-) diff --git a/0.notes.md b/0.notes.md index a106401e2..e1053e036 100644 --- a/0.notes.md +++ b/0.notes.md @@ -2,6 +2,8 @@ 2. on upgrade bump the coordinator index 3. on upgrade set the default api to empty -> force all API keys to be regenerated 4. on agent checkin, after upgrade, try to search api key by metadata.agentId and invalidate the "old" ones +5. ensure we delete the old API keys +6. fix painless script diff --git a/internal/pkg/dl/agent_integration_test.go b/internal/pkg/dl/agent_integration_test.go index f519d7320..039fa9f34 100644 --- a/internal/pkg/dl/agent_integration_test.go +++ b/internal/pkg/dl/agent_integration_test.go @@ -121,7 +121,7 @@ func TestFindAgent_NewModel(t *testing.T) { wantElasticsearchOutputs := map[string]*model.PolicyOutput{ "default": { APIKey: "TestFindNewModelAgent_APIKey", - APIKeyHistory: []model.APIKeyHistoryItems{ + ToRetireAPIKeys: []model.ToRetireAPIKeysItems{ { ID: "TestFindNewModelAgent_APIKeyID_invalidated", RetiredAt: "TestFindNewModelAgent_APIKeyID_invalidated_at"}, diff --git a/internal/pkg/model/schema.go b/internal/pkg/model/schema.go index a82d434d2..5b12479a5 100644 --- a/internal/pkg/model/schema.go +++ b/internal/pkg/model/schema.go @@ -25,16 +25,6 @@ func (d *ESDocument) ESInitialize(id string, seqno, version int64) { d.Version = version } -// APIKeyHistoryItems -type APIKeyHistoryItems struct { - - // API Key identifier - ID string `json:"id,omitempty"` - - // Date/time the API key was retired - RetiredAt string `json:"retired_at,omitempty"` -} - // Action An Elastic Agent action type Action struct { ESDocument @@ -134,7 +124,7 @@ type Agent struct { DefaultAPIKey string `json:"default_api_key,omitempty"` // Deprecated. Use Outputs instead. Default API Key History - DefaultAPIKeyHistory []APIKeyHistoryItems `json:"default_api_key_history,omitempty"` + DefaultAPIKeyHistory []ToRetireAPIKeysItems `json:"default_api_key_history,omitempty"` // Deprecated. Use Outputs instead. ID of the API key the Elastic Agent uses to authenticate with elasticsearch DefaultAPIKeyID string `json:"default_api_key_id,omitempty"` @@ -332,21 +322,21 @@ type PolicyLeader struct { Timestamp string `json:"@timestamp,omitempty"` } -// PolicyOutput +// PolicyOutput holds the needed data to manage the output API keys type PolicyOutput struct { ESDocument // API key the Elastic Agent uses to authenticate with elasticsearch APIKey string `json:"api_key"` - // Default API Key History - APIKeyHistory []APIKeyHistoryItems `json:"api_key_history"` - // ID of the API key the Elastic Agent uses to authenticate with elasticsearch APIKeyID string `json:"api_key_id"` // The policy output permissions hash PolicyPermissionsHash string `json:"policy_permissions_hash"` + + // API keys to be invalidated on next agent ack + ToRetireAPIKeys []ToRetireAPIKeysItems `json:"to_retire_api_keys,omitempty"` } // Server A Fleet Server @@ -370,6 +360,16 @@ type ServerMetadata struct { Version string `json:"version"` } +// ToRetireAPIKeysItems the Output API Keys that were replaced and should be retired +type ToRetireAPIKeysItems struct { + + // API Key identifier + ID string `json:"id,omitempty"` + + // Date/time the API key was retired + RetiredAt string `json:"retired_at,omitempty"` +} + // UserProvidedMetadata User provided metadata information for the Elastic Agent type UserProvidedMetadata struct { } diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index b22895a11..322d104b1 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -82,12 +82,22 @@ func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, agent.ElasticsearchOutputs[p.Name] = output } - // Migration path, use agent.ElasticsearchOutputs instead of agent.Default* + // Migration path: + // - force API keys to be regenerated: + // - make them empty + // - add them to Old API key, so they'll be deleted + // - use agent.ElasticsearchOutputs instead of agent.Default* if agent.DefaultAPIKey != "" { - output.APIKey = agent.DefaultAPIKey - output.APIKeyID = agent.DefaultAPIKeyID + output.APIKey = "" + output.APIKeyID = "" output.PolicyPermissionsHash = agent.PolicyOutputPermissionsHash - output.APIKeyHistory = agent.DefaultAPIKeyHistory + output.ToRetireAPIKeys = append(output.ToRetireAPIKeys, + model.ToRetireAPIKeysItems{ + ID: agent.DefaultAPIKeyID, + RetiredAt: time.Now().UTC().Format(time.RFC3339), + }) + output.ToRetireAPIKeys = append(output.ToRetireAPIKeys, + agent.DefaultAPIKeyHistory...) agent.DefaultAPIKey = "" agent.DefaultAPIKeyID = "" @@ -127,11 +137,12 @@ func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, ctx := zlog.WithContext(ctx) outputAPIKey, err := generateOutputAPIKey(ctx, bulker, agent.Id, p.Name, p.Role.Raw) if err != nil { - zlog.Error().Err(err).Msg("fail generate output key") - return err + return fmt.Errorf("failed generate output API key: %w", err) } output.APIKey = outputAPIKey.Agent() + output.APIKeyID = outputAPIKey.ID + output.PolicyPermissionsHash = p.Role.Sha2 // for the sake of consistency // When a new keys is generated we need to update the Agent record, // this will need to be updated when multiples Elasticsearch output @@ -146,8 +157,8 @@ func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, dl.FieldDefaultAPIKeyID: outputAPIKey.ID, dl.FieldPolicyOutputPermissionsHash: p.Role.Sha2, } - if agent.DefaultAPIKeyID != "" { // TODO: output.APIKeyID != "" - fields[dl.FieldDefaultAPIKeyHistory] = model.APIKeyHistoryItems{ + if output.APIKeyID != "" { + fields[dl.FieldDefaultAPIKeyHistory] = model.ToRetireAPIKeysItems{ ID: output.APIKeyID, RetiredAt: time.Now().UTC().Format(time.RFC3339), } diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go index 366737fba..876777f44 100644 --- a/internal/pkg/policy/policy_output_test.go +++ b/internal/pkg/policy/policy_output_test.go @@ -83,11 +83,15 @@ func TestPolicyESOutputPrepareNoRole(t *testing.T) { bulker.AssertExpectations(t) } -func TestPolicyOutputESPrepare(t *testing.T) { - t.Run("Permission hash == Agent Permission Hash no need to regenerate the key", func(t *testing.T) { +func TestPolicyOutputESPrepare_oldModel(t *testing.T) { + // TODO: ensure the DefaultAPIKeyHistory is copied to ElasticsearchOutputs[].ToRetireAPIKeys + // TODO: ensure current DefaultAPIKeyID id added to ElasticsearchOutputs[].ToRetireAPIKeys + t.Run("Permission hash == Agent Permission Hash -> force generate the key", func(t *testing.T) { logger := testlog.SetLogger(t) - bulker := ftesting.NewMockBulk() - apiKey := bulk.APIKey{ID: "test_id", Key: "EXISTING-KEY"} + + apiKey := bulk.APIKey{ID: "test_id_existing", Key: "existing-key"} + wantAPIKey := bulk.APIKey{ID: "test_id_new", Key: "new-key"} + hashPerm := "abc123" output := Output{ Type: OutputTypeElasticsearch, @@ -98,6 +102,14 @@ func TestPolicyOutputESPrepare(t *testing.T) { }, } + bulker := ftesting.NewMockBulk() + bulker.On("Update", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(nil).Once() + bulker.On("APIKeyCreate", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(&wantAPIKey, nil).Once() //nolint:govet // test case + policyMap := smap.Map{ "test output": map[string]interface{}{}, } @@ -105,11 +117,125 @@ func TestPolicyOutputESPrepare(t *testing.T) { testAgent := &model.Agent{ DefaultAPIKey: apiKey.Agent(), PolicyOutputPermissionsHash: hashPerm, + ElasticsearchOutputs: map[string]*model.PolicyOutput{}, + } + + err := output.Prepare(context.Background(), logger, bulker, testAgent, policyMap) + require.NoError(t, err, "expected prepare to pass") + + key, ok := policyMap.GetMap(output.Name)["api_key"].(string) + gotOutput := testAgent.ElasticsearchOutputs[output.Name] + + require.True(t, ok, "api key not present on policy map") + assert.Equal(t, wantAPIKey.Agent(), key) + + // Migration path: copy old values to new ElasticsearchOutputs field + assert.Equal(t, wantAPIKey.Agent(), gotOutput.APIKey) + assert.Equal(t, wantAPIKey.ID, gotOutput.APIKeyID) + assert.Equal(t, output.Role.Sha2, gotOutput.PolicyPermissionsHash) + + // Migration path: ensure Default* fields are left empty + assert.Empty(t, testAgent.DefaultAPIKey) + assert.Empty(t, testAgent.DefaultAPIKeyID) + assert.Empty(t, testAgent.DefaultAPIKeyHistory) + assert.Empty(t, testAgent.PolicyOutputPermissionsHash) + + bulker.AssertExpectations(t) + }) + + t.Run("Permission hash != Agent Permission Hash need to regenerate the key", func(t *testing.T) { + logger := testlog.SetLogger(t) + bulker := ftesting.NewMockBulk() + + oldAPIKey := bulk.APIKey{ID: "test_id", Key: "EXISTING-KEY"} + wantAPIKey := bulk.APIKey{ID: "abc", Key: "new-key"} + hashPerm := "old-HASH" + + bulker.On("Update", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(nil).Once() + bulker.On("APIKeyCreate", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(&wantAPIKey, nil).Once() //nolint:govet // test case + + output := Output{ + Type: OutputTypeElasticsearch, + Name: "test output", + Role: &RoleT{ + Sha2: "new-hash", + Raw: TestPayload, + }, + } + + policyMap := smap.Map{ + "test output": map[string]interface{}{}, + } + + testAgent := &model.Agent{ + DefaultAPIKey: oldAPIKey.Agent(), + PolicyOutputPermissionsHash: hashPerm, + ElasticsearchOutputs: map[string]*model.PolicyOutput{ + // output.Name: { + // ESDocument: model.ESDocument{}, + // APIKey: oldAPIKey.Agent(), + // ToRetireAPIKeys: nil, + // APIKeyID: oldAPIKey.ID, + // PolicyPermissionsHash: hashPerm, + // }, + }, + } + + err := output.Prepare(context.Background(), logger, bulker, testAgent, policyMap) + require.NoError(t, err, "expected prepare to pass") + + key, ok := policyMap.GetMap(output.Name)["api_key"].(string) + gotOutput := testAgent.ElasticsearchOutputs[output.Name] + + require.True(t, ok, "unable to case api key") + require.Equal(t, wantAPIKey.Agent(), key) + + // Migration path: copy old values to new ElasticsearchOutputs field + assert.Equal(t, wantAPIKey.Agent(), gotOutput.APIKey) + assert.Equal(t, wantAPIKey.ID, gotOutput.APIKeyID) + assert.Equal(t, output.Role.Sha2, gotOutput.PolicyPermissionsHash) + + // Migration path: ensure Default* fields are left empty + assert.Empty(t, testAgent.DefaultAPIKey) + assert.Empty(t, testAgent.DefaultAPIKeyID) + assert.Empty(t, testAgent.DefaultAPIKeyHistory) + assert.Empty(t, testAgent.PolicyOutputPermissionsHash) + + bulker.AssertExpectations(t) + }) +} + +func TestPolicyOutputESPrepare_newModel(t *testing.T) { + t.Run("Permission hash == Agent Permission Hash no need to regenerate the key", func(t *testing.T) { + logger := testlog.SetLogger(t) + bulker := ftesting.NewMockBulk() + + apiKey := bulk.APIKey{ID: "test_id_existing", Key: "existing-key"} + + hashPerm := "abc123" + output := Output{ + Type: OutputTypeElasticsearch, + Name: "test output", + Role: &RoleT{ + Sha2: hashPerm, + Raw: TestPayload, + }, + } + + policyMap := smap.Map{ + "test output": map[string]interface{}{}, + } + + testAgent := &model.Agent{ ElasticsearchOutputs: map[string]*model.PolicyOutput{ output.Name: { ESDocument: model.ESDocument{}, APIKey: apiKey.Agent(), - APIKeyHistory: nil, + ToRetireAPIKeys: nil, APIKeyID: apiKey.ID, PolicyPermissionsHash: hashPerm, }, @@ -120,10 +246,21 @@ func TestPolicyOutputESPrepare(t *testing.T) { require.NoError(t, err, "expected prepare to pass") key, ok := policyMap.GetMap(output.Name)["api_key"].(string) + gotOutput := testAgent.ElasticsearchOutputs[output.Name] - assert.True(t, ok, "api key not present on policy map") - assert.Equal(t, testAgent.ElasticsearchOutputs[output.Name].APIKey, key) - assert.Empty(t, testAgent.DefaultAPIKey) // Migration path: ensure we don't use DefaultAPIKey anymore + require.True(t, ok, "api key not present on policy map") + assert.Equal(t, apiKey.Agent(), key) + + assert.Equal(t, apiKey.Agent(), gotOutput.APIKey) + assert.Equal(t, apiKey.ID, gotOutput.APIKeyID) + assert.Equal(t, output.Role.Sha2, gotOutput.PolicyPermissionsHash) + assert.Empty(t, gotOutput.ToRetireAPIKeys) + + // Old model must always remain empty + assert.Empty(t, testAgent.DefaultAPIKey) + assert.Empty(t, testAgent.DefaultAPIKeyID) + assert.Empty(t, testAgent.DefaultAPIKeyHistory) + assert.Empty(t, testAgent.PolicyOutputPermissionsHash) bulker.AssertNotCalled(t, "Update", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) @@ -161,13 +298,11 @@ func TestPolicyOutputESPrepare(t *testing.T) { } testAgent := &model.Agent{ - DefaultAPIKey: oldAPIKey.Agent(), - PolicyOutputPermissionsHash: hashPerm, ElasticsearchOutputs: map[string]*model.PolicyOutput{ output.Name: { ESDocument: model.ESDocument{}, APIKey: oldAPIKey.Agent(), - APIKeyHistory: nil, + ToRetireAPIKeys: nil, APIKeyID: oldAPIKey.ID, PolicyPermissionsHash: hashPerm, }, @@ -178,11 +313,22 @@ func TestPolicyOutputESPrepare(t *testing.T) { require.NoError(t, err, "expected prepare to pass") key, ok := policyMap.GetMap(output.Name)["api_key"].(string) + gotOutput := testAgent.ElasticsearchOutputs[output.Name] require.True(t, ok, "unable to case api key") require.Equal(t, wantAPIKey.Agent(), key) - assert.Empty(t, testAgent.DefaultAPIKey) // Migration path: ensure we don't use DefaultAPIKey anymore - assert.Equal(t, testAgent.ElasticsearchOutputs[output.Name].APIKey, key) + + assert.Equal(t, wantAPIKey.Agent(), gotOutput.APIKey) + assert.Equal(t, wantAPIKey.ID, gotOutput.APIKeyID) + assert.Equal(t, output.Role.Sha2, gotOutput.PolicyPermissionsHash) + + // assert.Contains(t, gotOutput.ToRetireAPIKeys, oldAPIKey.ID) // TODO: assert on bulker.Update + + // Old model must always remain empty + assert.Empty(t, testAgent.DefaultAPIKey) + assert.Empty(t, testAgent.DefaultAPIKeyID) + assert.Empty(t, testAgent.DefaultAPIKeyHistory) + assert.Empty(t, testAgent.PolicyOutputPermissionsHash) bulker.AssertExpectations(t) }) @@ -217,9 +363,26 @@ func TestPolicyOutputESPrepare(t *testing.T) { require.NoError(t, err, "expected prepare to pass") key, ok := policyMap.GetMap(output.Name)["api_key"].(string) + gotOutput := testAgent.ElasticsearchOutputs[output.Name] require.True(t, ok, "unable to case api key") - require.Equal(t, apiKey.Agent(), key) + assert.Equal(t, apiKey.Agent(), key) + + assert.Equal(t, apiKey.Agent(), gotOutput.APIKey) + assert.Equal(t, apiKey.ID, gotOutput.APIKeyID) + assert.Equal(t, output.Role.Sha2, gotOutput.PolicyPermissionsHash) + assert.Empty(t, gotOutput.ToRetireAPIKeys) + + // Old model must always remain empty + assert.Empty(t, testAgent.DefaultAPIKey) + assert.Empty(t, testAgent.DefaultAPIKeyID) + assert.Empty(t, testAgent.DefaultAPIKeyHistory) + assert.Empty(t, testAgent.PolicyOutputPermissionsHash) + bulker.AssertExpectations(t) }) } + +func TestRenderUpdatePainlessScript(t *testing.T) { + +} diff --git a/model/schema.json b/model/schema.json index 0df7bba75..88aecc635 100644 --- a/model/schema.json +++ b/model/schema.json @@ -346,10 +346,10 @@ ] }, - "api_key_history": { - "description": "Output API Key History", + "to_retire_api_keys": { "type": "array", "items": { + "description": "the Output API Keys that were replaced and should be retired", "type": "object", "properties": { "id": { @@ -367,14 +367,15 @@ "policy_output" : { "type": "object", + "description": "holds the needed data to manage the output API keys", "properties": { "api_key": { "description": "API key the Elastic Agent uses to authenticate with elasticsearch", "type": "string" }, - "api_key_history": { - "description": "Default API Key History", - "$ref": "#/definitions/api_key_history" + "to_retire_api_keys": { + "description": "API keys to be invalidated on next agent ack", + "$ref": "#/definitions/to_retire_api_keys" }, "api_key_id": { "description": "ID of the API key the Elastic Agent uses to authenticate with elasticsearch", @@ -512,7 +513,7 @@ }, "default_api_key_history": { "description": "Deprecated. Use Outputs instead. Default API Key History", - "$ref": "#/definitions/api_key_history" + "$ref": "#/definitions/to_retire_api_keys" }, "elasticsearch_outputs": { "description": "ElasticsearchOutputs is the policy output data for each Elasticsearch output. It maps the output name to its data", From d492cfd8c43e4bccbd8e05ec25f27f01347f7263 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 25 Jul 2022 18:58:45 +0200 Subject: [PATCH 42/89] remove pre allocated agent.ElasticsearchOutputs --- internal/pkg/dl/agent.go | 8 +------- internal/pkg/policy/policy_output.go | 4 ++++ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/internal/pkg/dl/agent.go b/internal/pkg/dl/agent.go index 5b82cdb9d..a4871fa73 100644 --- a/internal/pkg/dl/agent.go +++ b/internal/pkg/dl/agent.go @@ -65,9 +65,6 @@ func FindAgent(ctx context.Context, bulker bulk.Bulk, tmpl *dsl.Tmpl, name strin return model.Agent{}, fmt.Errorf("could not unmarshal ES document into model.Agent: %w", err) } - if agent.ElasticsearchOutputs == nil { - agent.ElasticsearchOutputs = map[string]*model.PolicyOutput{} - } return agent, nil } @@ -91,10 +88,7 @@ func FindOfflineAgents(ctx context.Context, bulker bulk.Bulk, policyID string, u if err := hit.Unmarshal(&agents[i]); err != nil { return nil, fmt.Errorf("could not unmarshal ES document into model.Agent: %w", err) } - if agents[i].ElasticsearchOutputs == nil { - agents[i].ElasticsearchOutputs = map[string]*model.PolicyOutput{} - } } - + return agents, nil } diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index 322d104b1..729d38069 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -77,6 +77,10 @@ func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, output, ok := agent.ElasticsearchOutputs[p.Name] if !ok { + if agent.ElasticsearchOutputs == nil { + agent.ElasticsearchOutputs = map[string]*model.PolicyOutput{} + } + zlog.Debug().Msgf("creating agent.ElasticsearchOutputs[%s]", p.Name) output = &model.PolicyOutput{} agent.ElasticsearchOutputs[p.Name] = output From 35d11ec0dc7d24384378af3ba9fea6b8dd68c86e Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 27 Jul 2022 18:17:01 +0200 Subject: [PATCH 43/89] better errors, minor improvements --- internal/pkg/bulk/opBulk.go | 14 +++++++++----- internal/pkg/testing/esutil/bootstrap.go | 2 +- internal/pkg/testing/setup.go | 15 +++++++-------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/internal/pkg/bulk/opBulk.go b/internal/pkg/bulk/opBulk.go index 50b2c47e0..650962b86 100644 --- a/internal/pkg/bulk/opBulk.go +++ b/internal/pkg/bulk/opBulk.go @@ -7,6 +7,7 @@ package bulk import ( "bytes" "context" + "errors" "fmt" "time" @@ -187,7 +188,6 @@ func (b *Bulker) flushBulk(ctx context.Context, queue queueT) error { } res, err := req.Do(ctx, b.es) - if err != nil { log.Error().Err(err).Str("mod", kModBulk).Msg("Fail BulkRequest req.Do") return err @@ -217,12 +217,16 @@ func (b *Bulker) flushBulk(ctx context.Context, queue queueT) error { var blk bulkIndexerResponse blk.Items = make([]bulkStubItem, 0, queueCnt) + // TODO: We're loosing information abut the errors, we should check a way + // to return the full error ES returns if err = easyjson.Unmarshal(buf.Bytes(), &blk); err != nil { - log.Error(). - Err(err). + log.Err(err). Str("mod", kModBulk). - Msg("Unmarshal error") - return err + Msg("flushBulk failed, could not unmarshal ES response") + return fmt.Errorf("flushBulk failed, could not unmarshal ES response: %w", err) + } + if blk.HasErrors { + log.Debug().Err(errors.New(buf.String())).Msg("Bulk call: Es returned an error") } log.Trace(). diff --git a/internal/pkg/testing/esutil/bootstrap.go b/internal/pkg/testing/esutil/bootstrap.go index e2aafce76..978f95a75 100644 --- a/internal/pkg/testing/esutil/bootstrap.go +++ b/internal/pkg/testing/esutil/bootstrap.go @@ -10,7 +10,7 @@ import ( "github.com/elastic/go-elasticsearch/v7" ) -// EnsureIndex sets up the index if it doesn't exists, utilized for integration tests at the moment +// EnsureIndex sets up the index if it doesn't exist. It's utilized for integration tests at the moment. func EnsureIndex(ctx context.Context, cli *elasticsearch.Client, name, mapping string) error { err := EnsureTemplate(ctx, cli, name, mapping, false) if err != nil { diff --git a/internal/pkg/testing/setup.go b/internal/pkg/testing/setup.go index 8dac38cdc..6b317d126 100644 --- a/internal/pkg/testing/setup.go +++ b/internal/pkg/testing/setup.go @@ -98,7 +98,7 @@ func SetupCleanIndex(ctx context.Context, t *testing.T, index string, opts ...bu func CleanIndex(ctx context.Context, t *testing.T, bulker bulk.Bulk, index string) string { t.Helper() - t.Helper() + tmpl := dsl.NewTmpl() root := dsl.NewRoot() root.Query().MatchAll() @@ -106,7 +106,7 @@ func CleanIndex(ctx context.Context, t *testing.T, bulker bulk.Bulk, index strin query, err := q.Render(make(map[string]interface{})) if err != nil { - t.Fatal(err) + t.Fatalf("could not clena index: failed t render query template: %v", err) } cli := bulker.Client() @@ -114,17 +114,16 @@ func CleanIndex(ctx context.Context, t *testing.T, bulker bulk.Bulk, index strin cli.API.DeleteByQuery.WithContext(ctx), cli.API.DeleteByQuery.WithRefresh(true), ) - if err != nil { - t.Fatal(err) + t.Fatalf("could not clean index %s, DeleteByQuery failed: %v", + index, err) } defer res.Body.Close() var esres es.DeleteByQueryResponse - err = json.NewDecoder(res.Body).Decode(&esres) if err != nil { - t.Fatal(err) + t.Fatalf("could not decode ES response: %v", err) } if res.IsError() { @@ -135,9 +134,9 @@ func CleanIndex(ctx context.Context, t *testing.T, bulker bulk.Bulk, index strin } } } - if err != nil { - t.Fatal(err) + t.Fatalf("ES returned an error: %v. body: %q", err, res) } + return index } From 311e6b0af024e0340e85e08ce212968a12a976e6 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 27 Jul 2022 18:17:27 +0200 Subject: [PATCH 44/89] removed leftover test --- internal/pkg/dl/agent_integration_test.go | 28 ----------------------- 1 file changed, 28 deletions(-) diff --git a/internal/pkg/dl/agent_integration_test.go b/internal/pkg/dl/agent_integration_test.go index 039fa9f34..93646b531 100644 --- a/internal/pkg/dl/agent_integration_test.go +++ b/internal/pkg/dl/agent_integration_test.go @@ -152,31 +152,3 @@ func TestFindAgent_NewModel(t *testing.T) { assert.Equal(t, agentID, agent.Id) assert.Equal(t, wantElasticsearchOutputs, agent.ElasticsearchOutputs) } - -func TestFindAgent_ESOutput_never_nil(t *testing.T) { - index, bulker := ftesting.SetupCleanIndex(context.Background(), t, FleetAgents) - - now := time.Now().UTC().Format(time.RFC3339) - - policyID := uuid.Must(uuid.NewV4()).String() - agentID := uuid.Must(uuid.NewV4()).String() - - body, err := json.Marshal(model.Agent{ - PolicyID: policyID, - Active: true, - LastCheckin: now, - LastCheckinStatus: "", - UpdatedAt: now, - EnrolledAt: now, - }) - require.NoError(t, err) - - _, err = bulker.Create(context.Background(), index, agentID, body, bulk.WithRefresh()) - require.NoError(t, err) - - agent, err := FindAgent(context.Background(), bulker, QueryAgentByID, FieldID, agentID, WithIndexName(index)) - require.NoError(t, err) - - assert.Equal(t, agentID, agent.Id) - assert.NotNil(t, agent.ElasticsearchOutputs) -} From fc460acf8a0a3695e8e6f5a4414a884100ab8f86 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 27 Jul 2022 18:17:56 +0200 Subject: [PATCH 45/89] add root cause to ErrElastic --- internal/pkg/es/error.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/internal/pkg/es/error.go b/internal/pkg/es/error.go index 79b07499c..a5e575df5 100644 --- a/internal/pkg/es/error.go +++ b/internal/pkg/es/error.go @@ -37,17 +37,25 @@ func (e ErrElastic) Error() string { // Otherwise were getting: "elastic fail 404::" msg := "elastic fail " var b strings.Builder - b.Grow(len(msg) + 5 + len(e.Type) + len(e.Reason)) + b.Grow(len(msg) + 11 + len(e.Type) + len(e.Reason) + len(e.Cause.Type) + len(e.Cause.Reason)) b.WriteString(msg) b.WriteString(strconv.Itoa(e.Status)) if e.Type != "" { - b.WriteString(":") + b.WriteString(": ") b.WriteString(e.Type) } if e.Reason != "" { - b.WriteString(":") + b.WriteString(": ") b.WriteString(e.Reason) } + if e.Cause.Type != "" { + b.WriteString(": ") + b.WriteString(e.Cause.Type) + } + if e.Cause.Reason != "" { + b.WriteString(": ") + b.WriteString(e.Cause.Reason) + } return b.String() } @@ -83,8 +91,8 @@ func TranslateError(status int, e *ErrorT) error { Type string Reason string }{ - e.Cause.Type, - e.Cause.Reason, + Type: e.Cause.Type, + Reason: e.Cause.Reason, }, } } From 87a411ad21fc64f65e2b7d813c7e9f8352be727f Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 28 Jul 2022 17:50:57 +0200 Subject: [PATCH 46/89] more notes --- 0.notes.md | 3 +-- internal/pkg/dl/constants.go | 18 +++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/0.notes.md b/0.notes.md index e1053e036..6d0936313 100644 --- a/0.notes.md +++ b/0.notes.md @@ -4,8 +4,7 @@ 4. on agent checkin, after upgrade, try to search api key by metadata.agentId and invalidate the "old" ones 5. ensure we delete the old API keys 6. fix painless script - - +7. add output name or something to api key metadata diff --git a/internal/pkg/dl/constants.go b/internal/pkg/dl/constants.go index 14c5bc7a6..494981301 100644 --- a/internal/pkg/dl/constants.go +++ b/internal/pkg/dl/constants.go @@ -28,21 +28,21 @@ const ( FieldActionSeqNo = "action_seq_no" FieldActionID = "action_id" - FieldPolicyID = "policy_id" - FieldRevisionIdx = "revision_idx" + FieldAgent = "agent" + FieldAgentVersion = "version" FieldCoordinatorIdx = "coordinator_idx" FieldLastCheckin = "last_checkin" FieldLastCheckinStatus = "last_checkin_status" FieldLocalMetadata = "local_metadata" - FieldPolicyRevisionIdx = "policy_revision_idx" FieldPolicyCoordinatorIdx = "policy_coordinator_idx" - FieldDefaultAPIKey = "default_api_key" - FieldDefaultAPIKeyID = "default_api_key_id" //nolint:gosec // field name - FieldDefaultAPIKeyHistory = "default_api_key_history" //nolint:gosec // field name - FieldPolicyOutputPermissionsHash = "policy_output_permissions_hash" + FieldPolicyID = "policy_id" + FieldPolicyOutputAPIKey = "api_key" + FieldPolicyOutputAPIKeyID = "api_key_id" //nolint:gosec // field name + FieldPolicyOutputPermissionsHash = "policy_permissions_hash" + FieldPolicyOutputToRetireAPIKeys = "to_retire_api_keys" //nolint:gosec // field name + FieldPolicyRevisionIdx = "policy_revision_idx" + FieldRevisionIdx = "revision_idx" FieldUnenrolledReason = "unenrolled_reason" - FieldAgentVersion = "version" - FieldAgent = "agent" FieldActive = "active" FieldUpdatedAt = "updated_at" From e266c2079be4705db68c84592fe8da3d85cbbdde Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 28 Jul 2022 17:58:04 +0200 Subject: [PATCH 47/89] adjust painless script and add integration tests --- .../policy/policy_outpur_integration_test.go | 121 ++++++++++++++++++ internal/pkg/policy/policy_output.go | 30 +++-- internal/pkg/policy/policy_output_test.go | 4 - 3 files changed, 143 insertions(+), 12 deletions(-) create mode 100644 internal/pkg/policy/policy_outpur_integration_test.go diff --git a/internal/pkg/policy/policy_outpur_integration_test.go b/internal/pkg/policy/policy_outpur_integration_test.go new file mode 100644 index 000000000..839c435ec --- /dev/null +++ b/internal/pkg/policy/policy_outpur_integration_test.go @@ -0,0 +1,121 @@ +//go:build integration + +package policy + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/gofrs/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/fleet-server/v7/internal/pkg/bulk" + "github.com/elastic/fleet-server/v7/internal/pkg/dl" + "github.com/elastic/fleet-server/v7/internal/pkg/model" + ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing" +) + +func TestRenderUpdatePainlessScript(t *testing.T) { + tts := []struct { + name string + + existingToRetireAPIKeys []model.ToRetireAPIKeysItems + }{ + { + name: "to_retire_api_keys is empty", + }, + { + name: "to_retire_api_keys is not empty", + existingToRetireAPIKeys: []model.ToRetireAPIKeysItems{{ + ID: "pre_existing_ID", RetiredAt: "pre_existing__RetiredAt"}}, + }, + } + + for _, tt := range tts { + t.Run(tt.name, func(t *testing.T) { + outputPermissionSha := "new_permissionSHA_" + tt.name + outputName := "output_" + tt.name + outputAPIKey := bulk.APIKey{ID: "new_ID", Key: "new-key"} + + index, bulker := ftesting.SetupCleanIndex(context.Background(), t, dl.FleetAgents) + + now := time.Now().UTC() + nowStr := now.Format(time.RFC3339) + + agentID := uuid.Must(uuid.NewV4()).String() + policyID := uuid.Must(uuid.NewV4()).String() + + previousAPIKey := bulk.APIKey{ + ID: "old_" + outputAPIKey.ID, + Key: "old_" + outputAPIKey.Key, + } + + wantElasticsearchOutputs := map[string]*model.PolicyOutput{ + outputName: { + APIKey: outputAPIKey.Agent(), + APIKeyID: outputAPIKey.ID, + PolicyPermissionsHash: outputPermissionSha, + ToRetireAPIKeys: append(tt.existingToRetireAPIKeys, + model.ToRetireAPIKeysItems{ + ID: previousAPIKey.ID, RetiredAt: nowStr}), + }, + } + + agentModel := model.Agent{ + PolicyID: policyID, + Active: true, + LastCheckin: nowStr, + LastCheckinStatus: "", + UpdatedAt: nowStr, + EnrolledAt: nowStr, + ElasticsearchOutputs: map[string]*model.PolicyOutput{ + outputName: { + APIKey: previousAPIKey.Agent(), + APIKeyID: previousAPIKey.ID, + PolicyPermissionsHash: "old_" + outputPermissionSha, + }, + }, + } + if tt.existingToRetireAPIKeys != nil { + agentModel.ElasticsearchOutputs[outputName].ToRetireAPIKeys = + tt.existingToRetireAPIKeys + } + + body, err := json.Marshal(agentModel) + require.NoError(t, err) + + _, err = bulker.Create( + context.Background(), index, agentID, body, bulk.WithRefresh()) + require.NoError(t, err) + + fields := map[string]interface{}{ + dl.FieldPolicyOutputAPIKey: outputAPIKey.Agent(), + dl.FieldPolicyOutputAPIKeyID: outputAPIKey.ID, + dl.FieldPolicyOutputPermissionsHash: outputPermissionSha, + dl.FieldPolicyOutputToRetireAPIKeys: model.ToRetireAPIKeysItems{ + ID: previousAPIKey.ID, RetiredAt: nowStr}, + } + + got, err := renderUpdatePainlessScript(outputName, fields) + require.NoError(t, err, "renderUpdatePainlessScript returned an unexpected error") + + err = bulker.Update(context.Background(), dl.FleetAgents, agentID, got) + require.NoError(t, err, "bulker.Update failed") + + // there is some refresh thing that needs time, I didn't manage to find + // how ot fix it at the requests to ES level, thus this timeout here. + time.Sleep(time.Second) + + gotAgent, err := dl.FindAgent( + context.Background(), bulker, dl.QueryAgentByID, dl.FieldID, agentID, dl.WithIndexName(index)) + require.NoError(t, err) + + assert.Equal(t, agentID, gotAgent.Id) + assert.Len(t, gotAgent.ElasticsearchOutputs, len(wantElasticsearchOutputs)) + assert.Equal(t, wantElasticsearchOutputs, gotAgent.ElasticsearchOutputs) + }) + } +} diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index 729d38069..197dd098f 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -157,19 +157,19 @@ func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, Msg("Updating agent record to pick up default output key.") fields := map[string]interface{}{ - dl.FieldDefaultAPIKey: outputAPIKey.Agent(), - dl.FieldDefaultAPIKeyID: outputAPIKey.ID, + dl.FieldPolicyOutputAPIKey: outputAPIKey.Agent(), + dl.FieldPolicyOutputAPIKeyID: outputAPIKey.ID, dl.FieldPolicyOutputPermissionsHash: p.Role.Sha2, } if output.APIKeyID != "" { - fields[dl.FieldDefaultAPIKeyHistory] = model.ToRetireAPIKeysItems{ + fields[dl.FieldPolicyOutputToRetireAPIKeys] = model.ToRetireAPIKeysItems{ ID: output.APIKeyID, RetiredAt: time.Now().UTC().Format(time.RFC3339), } } // Using painless script to append the old keys to the history - body, err := renderUpdatePainlessScript(fields) + body, err := renderUpdatePainlessScript(p.Name, fields) if err != nil { return fmt.Errorf("could no tupdate painless script: %w", err) } @@ -206,13 +206,27 @@ func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, return nil } -func renderUpdatePainlessScript(fields map[string]interface{}) ([]byte, error) { +func renderUpdatePainlessScript(outputName string, fields map[string]interface{}) ([]byte, error) { var source strings.Builder + source.WriteString(fmt.Sprintf(` +if (ctx._source['elasticsearch_outputs']==null) + {ctx._source['elasticsearch_outputs']=new HashMap();} +if (ctx._source['elasticsearch_outputs']['%s']==null) + {ctx._source['elasticsearch_outputs']['%s']=new HashMap();} +`, + outputName, outputName)) + for field := range fields { - if field == dl.FieldDefaultAPIKeyHistory { - source.WriteString(fmt.Sprint("if (ctx._source.", field, "==null) {ctx._source.", field, "=new ArrayList();} ctx._source.", field, ".add(params.", field, ");")) + if field == dl.FieldPolicyOutputToRetireAPIKeys { + source.WriteString(fmt.Sprintf(` +if (ctx._source['elasticsearch_outputs']['%s'].%s==null) + {ctx._source['elasticsearch_outputs']['%s'].%s=new ArrayList();} +ctx._source['elasticsearch_outputs']['%s'].%s.add(params.%s); +`, outputName, field, outputName, field, outputName, field, field)) } else { - source.WriteString(fmt.Sprint("ctx._source.", field, "=", "params.", field, ";")) + source.WriteString(fmt.Sprintf(` +ctx._source['elasticsearch_outputs']['%s'].%s=params.%s;`, + outputName, field, field)) } } diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go index 876777f44..133b9e305 100644 --- a/internal/pkg/policy/policy_output_test.go +++ b/internal/pkg/policy/policy_output_test.go @@ -382,7 +382,3 @@ func TestPolicyOutputESPrepare_newModel(t *testing.T) { bulker.AssertExpectations(t) }) } - -func TestRenderUpdatePainlessScript(t *testing.T) { - -} From eba4f711f952b8a9dca36ec966bb2140352720ec Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 28 Jul 2022 18:06:49 +0200 Subject: [PATCH 48/89] fix typo --- ...tpur_integration_test.go => policy_output_integration_test.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename internal/pkg/policy/{policy_outpur_integration_test.go => policy_output_integration_test.go} (100%) diff --git a/internal/pkg/policy/policy_outpur_integration_test.go b/internal/pkg/policy/policy_output_integration_test.go similarity index 100% rename from internal/pkg/policy/policy_outpur_integration_test.go rename to internal/pkg/policy/policy_output_integration_test.go From 135ddcb72a139acde298b8497035ebc5cd6ecd8f Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 28 Jul 2022 18:12:05 +0200 Subject: [PATCH 49/89] tidy up --- internal/pkg/policy/policy_output.go | 43 ++++++++++++---------------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index 197dd098f..e94126cf7 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -41,8 +41,13 @@ type Output struct { // Prepare prepares the output p to be sent to the elastic-agent // The agent might be mutated for an elasticsearch output func (p *Output) Prepare(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, agent *model.Agent, outputMap smap.Map) error { + zlog = zlog.With(). + Str("fleet.agent.id", agent.Id). + Str("fleet.policy.output.name", p.Name).Logger() + switch p.Type { case OutputTypeElasticsearch: + zlog.Debug().Msg("preparing elasticsearch output") if err := p.prepareElasticsearch(ctx, zlog, bulker, agent, outputMap); err != nil { return fmt.Errorf("failed to prepare elasticsearch output %q: %w", p.Name, err) } @@ -56,13 +61,12 @@ func (p *Output) Prepare(ctx context.Context, zlog zerolog.Logger, bulker bulk.B return nil } -func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, agent *model.Agent, outputMap smap.Map) error { - zlog = zlog.With(). - Str("fleet.agent.id", agent.Id). - Str("fleet.policy.output.name", p.Name).Logger() - - zlog.Info().Msg("preparing elasticsearch output") - +func (p *Output) prepareElasticsearch( + ctx context.Context, + zlog zerolog.Logger, + bulker bulk.Bulk, + agent *model.Agent, + outputMap smap.Map) error { // The role is required to do api key management if p.Role == nil { zlog.Error(). @@ -71,10 +75,6 @@ func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, return ErrNoOutputPerms } - // 1 - just use the new ElasticsearchOutputs - // 2 - make the tests check if they're correctly filled in - // 3 - ensure Default* is made empty/nil at the end - output, ok := agent.ElasticsearchOutputs[p.Name] if !ok { if agent.ElasticsearchOutputs == nil { @@ -86,7 +86,7 @@ func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, agent.ElasticsearchOutputs[p.Name] = output } - // Migration path: + // Migration path, see https://github.com/elastic/fleet-server/issues/1672: // - force API keys to be regenerated: // - make them empty // - add them to Old API key, so they'll be deleted @@ -189,16 +189,6 @@ func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, // in place to reduce number of agent policy allocation when sending the updated // agent policy to multiple agents. // See: https://github.com/elastic/fleet-server/issues/1301 - // - // WIP: - // The agent struct is a pointer, thus shared between the runs of Prepare - // for each output. Given 2 outputs (1 and 2), if 1 gets a new API key, - // agent.DefaultAPIKey changes. However, if 2 does not get one, the code - // below runs anyway, therefore the API key for output 2 will be set to the - // api key for output 1 - // The agent struct cannot be shared! The new API key must be its own local - // variable and a special case for the 'default output' has to handle - // updating agent.DefaultAPIKey. if err := setMapObj(outputMap, output.APIKey, p.Name, "api_key"); err != nil { return err } @@ -208,22 +198,27 @@ func (p *Output) prepareElasticsearch(ctx context.Context, zlog zerolog.Logger, func renderUpdatePainlessScript(outputName string, fields map[string]interface{}) ([]byte, error) { var source strings.Builder + + // prepare agent.elasticsearch_outputs[OUTPUT_NAME] source.WriteString(fmt.Sprintf(` if (ctx._source['elasticsearch_outputs']==null) {ctx._source['elasticsearch_outputs']=new HashMap();} if (ctx._source['elasticsearch_outputs']['%s']==null) {ctx._source['elasticsearch_outputs']['%s']=new HashMap();} -`, - outputName, outputName)) +`, outputName, outputName)) for field := range fields { if field == dl.FieldPolicyOutputToRetireAPIKeys { + // dl.FieldPolicyOutputToRetireAPIKeys is a special case. + // It's an array that gets deleted when the keys are invalidated. + // Thus, append the old API key ID, create the field if necessary. source.WriteString(fmt.Sprintf(` if (ctx._source['elasticsearch_outputs']['%s'].%s==null) {ctx._source['elasticsearch_outputs']['%s'].%s=new ArrayList();} ctx._source['elasticsearch_outputs']['%s'].%s.add(params.%s); `, outputName, field, outputName, field, outputName, field, field)) } else { + // Update the other fields source.WriteString(fmt.Sprintf(` ctx._source['elasticsearch_outputs']['%s'].%s=params.%s;`, outputName, field, field)) From 89c920be0a1cee92b9ee57e241a66866fec988b6 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 28 Jul 2022 18:55:08 +0200 Subject: [PATCH 50/89] add output name to API key metadata --- internal/pkg/api/handleEnroll.go | 10 +- internal/pkg/apikey/apikey.go | 57 ++++++++++ .../pkg/apikey/apikey_integration_test.go | 103 ++++++++++++------ internal/pkg/apikey/get.go | 68 ------------ internal/pkg/apikey/metadata.go | 20 ++-- .../coordinator/monitor_integration_test.go | 8 +- internal/pkg/policy/policy_output.go | 12 +- 7 files changed, 160 insertions(+), 118 deletions(-) delete mode 100644 internal/pkg/apikey/get.go diff --git a/internal/pkg/api/handleEnroll.go b/internal/pkg/api/handleEnroll.go index d97943b53..7c5b1dd5a 100644 --- a/internal/pkg/api/handleEnroll.go +++ b/internal/pkg/api/handleEnroll.go @@ -186,7 +186,13 @@ func (et *EnrollerT) processRequest(rb *rollback.Rollback, zlog zerolog.Logger, return et._enroll(r.Context(), rb, zlog, req, erec.PolicyID, ver) } -func (et *EnrollerT) _enroll(ctx context.Context, rb *rollback.Rollback, zlog zerolog.Logger, req *EnrollRequest, policyID, ver string) (*EnrollResponse, error) { +func (et *EnrollerT) _enroll( + ctx context.Context, + rb *rollback.Rollback, + zlog zerolog.Logger, + req *EnrollRequest, + policyID, + ver string) (*EnrollResponse, error) { if req.SharedID != "" { // TODO: Support pre-existing install @@ -426,7 +432,7 @@ func generateAccessAPIKey(ctx context.Context, bulk bulk.Bulk, agentID string) ( agentID, "", []byte(kFleetAccessRolesJSON), - apikey.NewMetadata(agentID, apikey.TypeAccess), + apikey.NewMetadata(agentID, "", apikey.TypeAccess), ) } diff --git a/internal/pkg/apikey/apikey.go b/internal/pkg/apikey/apikey.go index 4924a647b..fccc26f3d 100644 --- a/internal/pkg/apikey/apikey.go +++ b/internal/pkg/apikey/apikey.go @@ -6,12 +6,17 @@ package apikey import ( + "context" "encoding/base64" + "encoding/json" "errors" "fmt" "net/http" "strings" "unicode/utf8" + + "github.com/elastic/go-elasticsearch/v7" + "github.com/elastic/go-elasticsearch/v7/esapi" ) const ( @@ -28,6 +33,58 @@ var ( var AuthKey = http.CanonicalHeaderKey("Authorization") +// APIKeyMetadata tracks Metadata associated with an APIKey. +type APIKeyMetadata struct { + ID string + Metadata Metadata +} + +// Read gathers APIKeyMetadata from Elasticsearch using the given client. +func Read(ctx context.Context, client *elasticsearch.Client, id string) (*APIKeyMetadata, error) { + opts := []func(*esapi.SecurityGetAPIKeyRequest){ + client.Security.GetAPIKey.WithContext(ctx), + client.Security.GetAPIKey.WithID(id), + } + + res, err := client.Security.GetAPIKey( + opts..., + ) + if err != nil { + return nil, fmt.Errorf("request to elasticsearch failed: %w", err) + } + defer res.Body.Close() + + if res.IsError() { + return nil, fmt.Errorf("%s: %w", res.String(), ErrAPIKeyNotFound) + } + + type APIKeyResponse struct { + ID string `json:"id"` + Metadata Metadata `json:"metadata"` + } + type GetAPIKeyResponse struct { + APIKeys []APIKeyResponse `json:"api_keys"` + } + + var resp GetAPIKeyResponse + d := json.NewDecoder(res.Body) + if err = d.Decode(&resp); err != nil { + return nil, fmt.Errorf( + "could not decode elasticsearch GetAPIKeyResponse: %w", err) + } + + if len(resp.APIKeys) == 0 { + return nil, ErrAPIKeyNotFound + } + + first := resp.APIKeys[0] + + return &APIKeyMetadata{ + ID: first.ID, + Metadata: first.Metadata, + }, nil +} + // APIKey is used to represent an Elasticsearch API Key. type APIKey struct { ID string diff --git a/internal/pkg/apikey/apikey_integration_test.go b/internal/pkg/apikey/apikey_integration_test.go index 5c4e3b97c..72f410d99 100644 --- a/internal/pkg/apikey/apikey_integration_test.go +++ b/internal/pkg/apikey/apikey_integration_test.go @@ -30,7 +30,7 @@ const testFleetRoles = ` } ` -func TestCreateAPIKeyWithMetadata(t *testing.T) { +func TestRead(t *testing.T) { ctx, cn := context.WithCancel(context.Background()) defer cn() @@ -44,44 +44,83 @@ func TestCreateAPIKeyWithMetadata(t *testing.T) { t.Fatal(err) } - // Create the key - agentID := uuid.Must(uuid.NewV4()).String() - name := uuid.Must(uuid.NewV4()).String() - akey, err := Create(ctx, es, name, "", "true", []byte(testFleetRoles), - NewMetadata(agentID, TypeAccess)) - if err != nil { - t.Fatal(err) + // Try to get the key that doesn't exist, expect ErrApiKeyNotFound + _, err = Read(ctx, es, "0000000000000") + if !errors.Is(err, ErrAPIKeyNotFound) { + t.Errorf("Unexpected error type: %v", err) } - // Get the key and verify that metadata was saved correctly - aKeyMeta, err := Read(ctx, es, akey.ID) - if err != nil { - t.Fatal(err) +} +func TestCreateAPIKeyWithMetadata(t *testing.T) { + tts := []struct { + name string + outputName string + }{ + {name: "with metadata.output_name", outputName: "a_output_name"}, + {name: "without metadata.output_name"}, } - diff := cmp.Diff(ManagedByFleetServer, aKeyMeta.Metadata.ManagedBy) - if diff != "" { - t.Error(diff) - } + for _, tt := range tts { + t.Run(tt.name, func(t *testing.T) { + ctx, cn := context.WithCancel(context.Background()) + defer cn() - diff = cmp.Diff(true, aKeyMeta.Metadata.Managed) - if diff != "" { - t.Error(diff) - } + cfg := elasticsearch.Config{ + Username: "elastic", + Password: "changeme", + } - diff = cmp.Diff(agentID, aKeyMeta.Metadata.AgentID) - if diff != "" { - t.Error(diff) - } + es, err := elasticsearch.NewClient(cfg) + if err != nil { + t.Fatal(err) + } - diff = cmp.Diff(TypeAccess.String(), aKeyMeta.Metadata.Type) - if diff != "" { - t.Error(diff) - } + // Create the API key + agentID := uuid.Must(uuid.NewV4()).String() + name := uuid.Must(uuid.NewV4()).String() + outputName := tt.outputName + apiKey, err := Create( + ctx, + es, + name, + "", + "true", + []byte(testFleetRoles), + NewMetadata(agentID, outputName, TypeAccess)) + if err != nil { + t.Fatal(err) + } - // Try to get the key that doesn't exists, expect ErrApiKeyNotFound - _, err = Read(ctx, es, "0000000000000") - if !errors.Is(err, ErrAPIKeyNotFound) { - t.Errorf("Unexpected error type: %v", err) + // Get the API key and verify that the metadata was saved correctly + aKeyMeta, err := Read(ctx, es, apiKey.ID) + if err != nil { + t.Fatal(err) + } + + diff := cmp.Diff(ManagedByFleetServer, aKeyMeta.Metadata.ManagedBy) + if diff != "" { + t.Error(diff) + } + + diff = cmp.Diff(true, aKeyMeta.Metadata.Managed) + if diff != "" { + t.Error(diff) + } + + diff = cmp.Diff(agentID, aKeyMeta.Metadata.AgentID) + if diff != "" { + t.Error(diff) + } + + diff = cmp.Diff(outputName, aKeyMeta.Metadata.OutputName) + if diff != "" { + t.Error(diff) + } + + diff = cmp.Diff(TypeAccess.String(), aKeyMeta.Metadata.Type) + if diff != "" { + t.Error(diff) + } + }) } } diff --git a/internal/pkg/apikey/get.go b/internal/pkg/apikey/get.go deleted file mode 100644 index 5d931c670..000000000 --- a/internal/pkg/apikey/get.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package apikey - -import ( - "context" - "encoding/json" - - "github.com/elastic/go-elasticsearch/v7" - "github.com/elastic/go-elasticsearch/v7/esapi" - "github.com/pkg/errors" -) - -// APIKetMetadata tracks Metadata associated with an APIKey. -type APIKeyMetadata struct { - ID string - Metadata Metadata -} - -// Read gathers APIKeyMetadata from Elasticsearch using the given client. -func Read(ctx context.Context, client *elasticsearch.Client, id string) (*APIKeyMetadata, error) { - - opts := []func(*esapi.SecurityGetAPIKeyRequest){ - client.Security.GetAPIKey.WithContext(ctx), - client.Security.GetAPIKey.WithID(id), - } - - res, err := client.Security.GetAPIKey( - opts..., - ) - - if err != nil { - return nil, err - } - defer res.Body.Close() - - if res.IsError() { - err = errors.Wrap(ErrAPIKeyNotFound, res.String()) - return nil, err - } - - type APIKeyResponse struct { - ID string `json:"id"` - Metadata Metadata `json:"metadata"` - } - type GetAPIKeyResponse struct { - APIKeys []APIKeyResponse `json:"api_keys"` - } - - var resp GetAPIKeyResponse - d := json.NewDecoder(res.Body) - if err = d.Decode(&resp); err != nil { - return nil, err - } - - if len(resp.APIKeys) == 0 { - return nil, ErrAPIKeyNotFound - } - - first := resp.APIKeys[0] - - return &APIKeyMetadata{ - ID: first.ID, - Metadata: first.Metadata, - }, nil -} diff --git a/internal/pkg/apikey/metadata.go b/internal/pkg/apikey/metadata.go index c80997c7b..d00380c01 100644 --- a/internal/pkg/apikey/metadata.go +++ b/internal/pkg/apikey/metadata.go @@ -19,18 +19,20 @@ func (t Type) String() string { // Metadata is additional information associated with an APIKey. type Metadata struct { - AgentID string `json:"agent_id,omitempty"` - Managed bool `json:"managed,omitempty"` - ManagedBy string `json:"managed_by,omitempty"` - Type string `json:"type,omitempty"` + AgentID string `json:"agent_id,omitempty"` + Managed bool `json:"managed,omitempty"` + ManagedBy string `json:"managed_by,omitempty"` + OutputName string `json:"output_name,omitempty"` + Type string `json:"type,omitempty"` } // NewMetadata returns Metadata for the given agentID. -func NewMetadata(agentID string, typ Type) Metadata { +func NewMetadata(agentID string, outputName string, typ Type) Metadata { return Metadata{ - AgentID: agentID, - Managed: true, - ManagedBy: ManagedByFleetServer, - Type: typ.String(), + AgentID: agentID, + Managed: true, + ManagedBy: ManagedByFleetServer, + OutputName: outputName, + Type: typ.String(), } } diff --git a/internal/pkg/coordinator/monitor_integration_test.go b/internal/pkg/coordinator/monitor_integration_test.go index 10d1f8960..a43e3c1f0 100644 --- a/internal/pkg/coordinator/monitor_integration_test.go +++ b/internal/pkg/coordinator/monitor_integration_test.go @@ -159,7 +159,7 @@ func TestMonitorUnenroller(t *testing.T) { agentID, "", []byte(""), - apikey.NewMetadata(agentID, apikey.TypeAccess), + apikey.NewMetadata(agentID, "", apikey.TypeAccess), ) require.NoError(t, err) outputKey, err := bulker.APIKeyCreate( @@ -167,7 +167,7 @@ func TestMonitorUnenroller(t *testing.T) { agentID, "", []byte(""), - apikey.NewMetadata(agentID, apikey.TypeAccess), + apikey.NewMetadata(agentID, "default", apikey.TypeAccess), ) require.NoError(t, err) @@ -306,7 +306,7 @@ func TestMonitorUnenrollerSetAndClear(t *testing.T) { agentID, "", []byte(""), - apikey.NewMetadata(agentID, apikey.TypeAccess), + apikey.NewMetadata(agentID, "", apikey.TypeAccess), ) require.NoError(t, err) outputKey, err := bulker.APIKeyCreate( @@ -314,7 +314,7 @@ func TestMonitorUnenrollerSetAndClear(t *testing.T) { agentID, "", []byte(""), - apikey.NewMetadata(agentID, apikey.TypeAccess), + apikey.NewMetadata(agentID, "default", apikey.TypeAccess), ) require.NoError(t, err) diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index e94126cf7..95cca624b 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -139,7 +139,8 @@ func (p *Output) prepareElasticsearch( Msg("Generating a new API key") ctx := zlog.WithContext(ctx) - outputAPIKey, err := generateOutputAPIKey(ctx, bulker, agent.Id, p.Name, p.Role.Raw) + outputAPIKey, err := + generateOutputAPIKey(ctx, bulker, agent.Id, p.Name, p.Role.Raw) if err != nil { return fmt.Errorf("failed generate output API key: %w", err) } @@ -236,7 +237,12 @@ ctx._source['elasticsearch_outputs']['%s'].%s=params.%s;`, return body, err } -func generateOutputAPIKey(ctx context.Context, bulk bulk.Bulk, agentID, outputName string, roles []byte) (*apikey.APIKey, error) { +func generateOutputAPIKey( + ctx context.Context, + bulk bulk.Bulk, + agentID, + outputName string, + roles []byte) (*apikey.APIKey, error) { name := fmt.Sprintf("%s:%s", agentID, outputName) zerolog.Ctx(ctx).Info().Msgf("generating output API key %s for agent ID %s", name, agentID) @@ -245,7 +251,7 @@ func generateOutputAPIKey(ctx context.Context, bulk bulk.Bulk, agentID, outputNa name, "", roles, - apikey.NewMetadata(agentID, apikey.TypeOutput), + apikey.NewMetadata(agentID, outputName, apikey.TypeOutput), ) } From c4cc70ee999b7c36da6e41518e69f84468a17d70 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Fri, 29 Jul 2022 16:30:15 +0200 Subject: [PATCH 51/89] add migration to migration.go --- cmd/fleet/main.go | 5 +- internal/pkg/dl/migration.go | 175 ++++++++++++------ internal/pkg/dl/migration_integration_test.go | 111 +++++++++++ internal/pkg/policy/policy_output.go | 3 + internal/pkg/testing/setup.go | 20 ++ 5 files changed, 253 insertions(+), 61 deletions(-) create mode 100644 internal/pkg/dl/migration_integration_test.go diff --git a/cmd/fleet/main.go b/cmd/fleet/main.go index 2f6be9a9e..2838896a7 100644 --- a/cmd/fleet/main.go +++ b/cmd/fleet/main.go @@ -809,12 +809,12 @@ func (f *FleetServer) runSubsystems(ctx context.Context, cfg *config.Config, g * return fmt.Errorf("failed version compatibility check with elasticsearch: %w", err) } - // Run migrations; current safe to do in background. That may change in the future. + // Run migrations; currently it's safe to do it in the background. That may change in the future. g.Go(loggedRunFunc(ctx, "Migrations", func(ctx context.Context) error { return dl.Migrate(ctx, bulker) })) - // Run schduler for periodic GC/cleanup + // Run scheduler for periodic GC/cleanup gcCfg := cfg.Inputs[0].Server.GC sched, err := scheduler.New(gc.Schedules(bulker, gcCfg.ScheduleInterval, gcCfg.CleanupAfterExpiredInterval)) if err != nil { @@ -831,6 +831,7 @@ func (f *FleetServer) runSubsystems(ctx context.Context, cfg *config.Config, g * } // Coordinator policy monitor + // TODO(Anderson): perhapes here to increase the coordinator index pim, err := monitor.New(dl.FleetPolicies, esCli, monCli, monitor.WithFetchSize(cfg.Inputs[0].Monitor.FetchSize), monitor.WithPollTimeout(cfg.Inputs[0].Monitor.PollTimeout), diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 4beb26741..5b6a24ee6 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -20,51 +20,36 @@ import ( "github.com/rs/zerolog/log" ) -func Migrate(ctx context.Context, bulker bulk.Bulk) error { - return migrateAgentMetadata(ctx, bulker) -} - -// FleetServer 7.15 added a new *AgentMetadata field to the Agent record. -// This field was populated in new enrollments in 7.15 and later; however, the -// change was not backported to support 7.14. The security team is reliant on the -// existence of this field in 7.16, so the following migration was added to -// support upgrade from 7.14. -// -// It is currently safe to run this in the background; albeit with some -// concern on conflicts. The conflict risk exists regardless as N Fleet Servers -// can be run in parallel at the same time. -// -// As the update only occurs once, the 99.9% case is a noop. -func migrateAgentMetadata(ctx context.Context, bulker bulk.Bulk) error { - - root := dsl.NewRoot() - root.Query().Bool().MustNot().Exists("agent.id") - - painless := "ctx._source.agent = [:]; ctx._source.agent.id = ctx._id;" - root.Param("script", painless) - - body, err := root.MarshalJSON() - if err != nil { - return err +type ( + migrationBodyFn func() (string, []byte, error) + migrationResponse struct { + Took int `json:"took"` + TimedOut bool `json:"timed_out"` + Total int `json:"total"` + Updated int `json:"updated"` + Deleted int `json:"deleted"` + Batches int `json:"batches"` + VersionConflicts int `json:"version_conflicts"` + Noops int `json:"noops"` + Retries struct { + Bulk int `json:"bulk"` + Search int `json:"search"` + } `json:"retries"` + Failures []json.RawMessage `json:"failures"` } +) -LOOP: - for { - nConflicts, err := updateAgentMetadata(ctx, bulker, body) - if err != nil { +func Migrate(ctx context.Context, bulker bulk.Bulk) error { + for _, fn := range []migrationBodyFn{migrateAgentMetadata, migrateElasticsearchOutputs} { + if _, err := migrate(ctx, bulker, fn); err != nil { return err } - if nConflicts == 0 { - break LOOP - } - - time.Sleep(time.Second) } return nil } -func updateAgentMetadata(ctx context.Context, bulker bulk.Bulk, body []byte) (int, error) { +func applyMigration(ctx context.Context, name string, bulker bulk.Bulk, body []byte) (migrationResponse, error) { start := time.Now() client := bulker.Client() @@ -79,39 +64,25 @@ func updateAgentMetadata(ctx context.Context, bulker bulk.Bulk, body []byte) (in } res, err := client.UpdateByQuery([]string{FleetAgents}, opts...) - if err != nil { - return 0, err + return migrationResponse{}, err } if res.IsError() { if res.StatusCode == http.StatusNotFound { // Ignore index not created yet; nothing to upgrade - return 0, nil + return migrationResponse{}, nil } - return 0, fmt.Errorf("Migrate UpdateByQuery %s", res.String()) + return migrationResponse{}, fmt.Errorf("migrate %s UpdateByQuery failed: %s", + name, res.String()) } - resp := struct { - Took int `json:"took"` - TimedOut bool `json:"timed_out"` - Total int `json:"total"` - Updated int `json:"updated"` - Deleted int `json:"deleted"` - Batches int `json:"batches"` - VersionConflicts int `json:"version_conflicts"` - Noops int `json:"noops"` - Retries struct { - Bulk int `json:"bulk"` - Search int `json:"search"` - } `json:"retries"` - Failures []json.RawMessage `json:"failures"` - }{} + resp := migrationResponse{} decoder := json.NewDecoder(res.Body) if err := decoder.Decode(&resp); err != nil { - return 0, errors.Wrap(err, "decode UpdateByQuery response") + return migrationResponse{}, errors.Wrap(err, "decode UpdateByQuery response") } log.Info(). @@ -126,11 +97,97 @@ func updateAgentMetadata(ctx context.Context, bulker bulk.Bulk, body []byte) (in Int("retries.bulk", resp.Retries.Bulk). Int("retries.search", resp.Retries.Search). Dur("rtt", time.Since(start)). - Msg("migrate agent records response") + Msgf("migration %s: agent records response", name) for _, fail := range resp.Failures { - log.Error().RawJSON("failure", fail).Msg("migration failure") + log.Error().RawJSON("failure", fail).Msgf("failed applying %s migration", name) + } + + return resp, err +} + +func migrate(ctx context.Context, bulker bulk.Bulk, fn migrationBodyFn) (int, error) { + var updatedDocs int + for { + name, body, err := fn() + if err != nil { + return updatedDocs, fmt.Errorf(": %w", err) + } + + resp, err := applyMigration(ctx, name, bulker, body) + if err != nil { + return updatedDocs, fmt.Errorf("failed to apply migration %q: %w", + name, err) + } + updatedDocs += resp.Updated + if resp.VersionConflicts == 0 { + break + } + + time.Sleep(time.Second) + } + return updatedDocs, nil +} + +// FleetServer 7.15 added a new *AgentMetadata field to the Agent record. +// This field was populated in new enrollments in 7.15 and later; however, the +// change was not backported to support 7.14. The security team is reliant on the +// existence of this field in 7.16, so the following migration was added to +// support upgrade from 7.14. +// +// It is currently safe to run this in the background; albeit with some +// concern on conflicts. The conflict risk exists regardless as N Fleet Servers +// can be run in parallel at the same time. +// +// As the update only occurs once, the 99.9% case is a noop. +func migrateAgentMetadata() (string, []byte, error) { + const migrationName = "AgentMetadata" + root := dsl.NewRoot() + root.Query().Bool().MustNot().Exists("agent.id") + + painless := "ctx._source.agent = [:]; ctx._source.agent.id = ctx._id;" + root.Param("script", painless) + + body, err := root.MarshalJSON() + if err != nil { + return migrationName, nil, fmt.Errorf("could not marshal ES query: %w", err) + } + + return migrationName, body, nil +} + +// TODO(Anderson): Add migration description +func migrateElasticsearchOutputs() (string, []byte, error) { + const migrationName = "ElasticsearchOutputs" + + root := dsl.NewRoot() + root.Query().Bool().MustNot().Exists("elasticsearch_outputs") + + painless := ` +// set up the new filed +if (ctx._source['elasticsearch_outputs']==null) + {ctx._source['elasticsearch_outputs']=new HashMap();} +if (ctx._source['elasticsearch_outputs']['default']==null) + {ctx._source['elasticsearch_outputs']['default']=new HashMap();} + +// copy old values to new 'elasticsearch_outputs' field +ctx._source['elasticsearch_outputs']['default'].to_retire_api_keys=ctx._source.default_api_key_history; +ctx._source['elasticsearch_outputs']['default'].api_key=ctx._source.default_api_key; +ctx._source['elasticsearch_outputs']['default'].api_key_id=ctx._source.default_api_key_id; +ctx._source['elasticsearch_outputs']['default'].policy_permissions_hash=ctx._source.policy_output_permissions_hash; + +// Erase deprecated fields +ctx._source.default_api_key_history=null; +ctx._source.default_api_key=""; +ctx._source.default_api_key_id=""; +ctx._source.policy_output_permissions_hash=""; +` + root.Param("script", painless) + + body, err := root.MarshalJSON() + if err != nil { + return migrationName, nil, fmt.Errorf("could not marshal ES query: %w", err) } - return resp.VersionConflicts, err + return migrationName, body, nil } diff --git a/internal/pkg/dl/migration_integration_test.go b/internal/pkg/dl/migration_integration_test.go new file mode 100644 index 000000000..8e7b996a9 --- /dev/null +++ b/internal/pkg/dl/migration_integration_test.go @@ -0,0 +1,111 @@ +//go:build integration + +package dl + +import ( + "context" + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/gofrs/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/fleet-server/v7/internal/pkg/bulk" + "github.com/elastic/fleet-server/v7/internal/pkg/model" + ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing" +) + +func createSomeAgents(t *testing.T, n int, apiKey bulk.APIKey, index string, bulker bulk.Bulk) []string { + t.Helper() + + var createdAgents []string + + for i := 0; i < n; i++ { + outputAPIKey := bulk.APIKey{ + ID: fmt.Sprint(apiKey.ID, i), + Key: fmt.Sprint(apiKey.Key, i), + } + + now := time.Now().UTC() + nowStr := now.Format(time.RFC3339) + + agentID := uuid.Must(uuid.NewV4()).String() + policyID := uuid.Must(uuid.NewV4()).String() + + agentModel := model.Agent{ + PolicyID: policyID, + Active: true, + LastCheckin: nowStr, + LastCheckinStatus: "", + UpdatedAt: nowStr, + EnrolledAt: nowStr, + DefaultAPIKeyID: outputAPIKey.ID, + DefaultAPIKey: outputAPIKey.Agent(), + PolicyOutputPermissionsHash: fmt.Sprint("a_output_permission_SHA_", i), + DefaultAPIKeyHistory: []model.ToRetireAPIKeysItems{ + { + ID: "old_" + outputAPIKey.ID, + RetiredAt: now.Add(-5 * time.Minute).Format(time.RFC3339), + }, + }, + } + + body, err := json.Marshal(agentModel) + require.NoError(t, err) + + _, err = bulker.Create( + context.Background(), index, agentID, body, bulk.WithRefresh()) + require.NoError(t, err) + + createdAgents = append(createdAgents, agentID) + } + + return createdAgents +} + +func TestMigrateElasticsearchOutputs(t *testing.T) { + index, bulker := ftesting.SetupCleanIndex(context.Background(), t, FleetAgents) + apiKey := bulk.APIKey{ + ID: fmt.Sprint("testAgent_"), + Key: fmt.Sprint("testAgent_key_"), + } + + agentIDs := createSomeAgents(t, 10, apiKey, index, bulker) + + migratedAgents, err := migrate(context.Background(), bulker, migrateElasticsearchOutputs) + require.NoError(t, err) + + assert.Equal(t, len(agentIDs), migratedAgents) + + for i, id := range agentIDs { + wantAPIKey := bulk.APIKey{ + ID: fmt.Sprint(apiKey.ID, i), + Key: fmt.Sprint(apiKey.Key, i), + } + + got, err := FindAgent( + context.Background(), bulker, QueryAgentByID, FieldID, id, WithIndexName(index)) + if err != nil { + assert.NoError(t, err, "failed to find agent ID %q", id) // we want to continue even if a single agent fails + continue + } + + // Assert new fields + require.Len(t, got.ElasticsearchOutputs, 1) + assert.Equal(t, wantAPIKey.Agent(), got.ElasticsearchOutputs["default"].APIKey) + assert.Equal(t, wantAPIKey.ID, got.ElasticsearchOutputs["default"].APIKeyID) + assert.Equal(t, wantAPIKey.Agent(), got.ElasticsearchOutputs["default"].APIKey) + assert.Equal(t, + fmt.Sprint("a_output_permission_SHA_", i), + got.ElasticsearchOutputs["default"].PolicyPermissionsHash) + + // Assert deprecated fields + assert.Empty(t, got.DefaultAPIKey) + assert.Empty(t, got.DefaultAPIKey) + assert.Empty(t, got.PolicyOutputPermissionsHash) + assert.Nil(t, got.DefaultAPIKeyHistory) + } +} diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index 95cca624b..790e6b5d2 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -86,6 +86,9 @@ func (p *Output) prepareElasticsearch( agent.ElasticsearchOutputs[p.Name] = output } + // Now that there is a 'migrateElasticsearchOutputs' on internal/pkg/dl/migration.go, + // the code below likely can be deleted and this function can only rely on the new agent model. + // Migration path, see https://github.com/elastic/fleet-server/issues/1672: // - force API keys to be regenerated: // - make them empty diff --git a/internal/pkg/testing/setup.go b/internal/pkg/testing/setup.go index 6b317d126..d7a1aa27d 100644 --- a/internal/pkg/testing/setup.go +++ b/internal/pkg/testing/setup.go @@ -12,6 +12,7 @@ import ( "context" "encoding/json" "errors" + "os" "testing" "github.com/elastic/go-elasticsearch/v7" @@ -37,6 +38,12 @@ fleet: `) func init() { + os.Setenv("ELASTICSEARCH_SERVICE_TOKEN", + "AAEAAWVsYXN0aWMvZmxlZXQtc2VydmVyL3Rva2VuMToyNzJ5dmhndFM3UzVsb1h3SERhT0dB") + os.Setenv("ELASTICSEARCH_HOSTS", "localhost:9200") + os.Setenv("ELASTICSEARCH_USERNAME", "elastic") + os.Setenv("ELASTICSEARCH_PASSWORD", "changeme") + c, err := yaml.NewConfig(defaultCfgData, config.DefaultOptions...) if err != nil { panic(err) @@ -110,6 +117,19 @@ func CleanIndex(ctx context.Context, t *testing.T, bulker bulk.Bulk, index strin } cli := bulker.Client() + + // // Just running DeleteByQuery with refresh true, does not seem to be enought + // req, err := http.NewRequest(http.MethodPost, index+"/_refresh", nil) + // if err != nil { + // t.Fatalf("could not clean index: failed to create request to refresh index %q: %v", + // index, err) + // } + // _, err = cli.Perform(req) + // if err != nil { + // t.Fatalf("could not clean index: failed to refresh index %q: %v", + // index, err) + // } + res, err := cli.API.DeleteByQuery([]string{index}, bytes.NewReader(query), cli.API.DeleteByQuery.WithContext(ctx), cli.API.DeleteByQuery.WithRefresh(true), From 21b3984223a223a5236d36ae323ae76f35a79c40 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 1 Aug 2022 07:47:33 +0200 Subject: [PATCH 52/89] add migration docs and improve logging --- internal/pkg/dl/migration.go | 38 +++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 5b6a24ee6..dcab89f8f 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -86,18 +86,19 @@ func applyMigration(ctx context.Context, name string, bulker bulk.Bulk, body []b } log.Info(). - Int("took", resp.Took). - Bool("timed_out", resp.TimedOut). - Int("total", resp.Total). - Int("updated", resp.Updated). - Int("deleted", resp.Deleted). - Int("batches", resp.Batches). - Int("version_conflicts", resp.VersionConflicts). - Int("noops", resp.Noops). - Int("retries.bulk", resp.Retries.Bulk). - Int("retries.search", resp.Retries.Search). - Dur("rtt", time.Since(start)). - Msgf("migration %s: agent records response", name) + Str("fleet.migration.name", name). + Int("fleet.migration.es.took", resp.Took). + Bool("fleet.migration.es.timed_out", resp.TimedOut). + Int("fleet.migration.total", resp.Total). + Int("fleet.migration.updated", resp.Updated). + Int("fleet.migration.deleted", resp.Deleted). + Int("fleet.migration.batches", resp.Batches). + Int("fleet.migration.version_conflicts", resp.VersionConflicts). + Int("fleet.migration.noops", resp.Noops). + Int("fleet.migration.retries.bulk", resp.Retries.Bulk). + Int("fleet.migration.retries.search", resp.Retries.Search). + Dur("fleet.migration.total.duration", time.Since(start)). + Msgf("migration %s done", name) for _, fail := range resp.Failures { log.Error().RawJSON("failure", fail).Msgf("failed applying %s migration", name) @@ -136,7 +137,7 @@ func migrate(ctx context.Context, bulker bulk.Bulk, fn migrationBodyFn) (int, er // support upgrade from 7.14. // // It is currently safe to run this in the background; albeit with some -// concern on conflicts. The conflict risk exists regardless as N Fleet Servers +// concern on conflicts. The conflict risk exists regardless as N Fleet Servers // can be run in parallel at the same time. // // As the update only occurs once, the 99.9% case is a noop. @@ -156,7 +157,16 @@ func migrateAgentMetadata() (string, []byte, error) { return migrationName, body, nil } -// TODO(Anderson): Add migration description +// FleetServer 8.4.0 introduces a new field to the Agent document, Outputs, to +// store the outputs credentials and data. The DefaultAPIKey, DefaultAPIKeyID, +// DefaultAPIKeyHistory and PolicyOutputPermissionsHash are now deprecated in +// favour of the new `Outputs` fields, which maps the output name to its data. +// This change fixes https://github.com/elastic/fleet-server/issues/1672. + +// The change is backward compatible as the deprecated fields are just set to +// their zero value and an older version of FleetServer can repopulate them. +// However, reverting FleetServer to an older version might cause very issue +// this change fixes. func migrateElasticsearchOutputs() (string, []byte, error) { const migrationName = "ElasticsearchOutputs" From 935706af86397a79809497934a2bc42a9210fca9 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 1 Aug 2022 08:05:38 +0200 Subject: [PATCH 53/89] remove migration code from policy_output --- internal/pkg/policy/policy_output.go | 40 +------ internal/pkg/policy/policy_output_test.go | 128 +--------------------- 2 files changed, 7 insertions(+), 161 deletions(-) diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index 790e6b5d2..2a29bef1d 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -70,7 +70,6 @@ func (p *Output) prepareElasticsearch( // The role is required to do api key management if p.Role == nil { zlog.Error(). - Str("fleet.output.name", p.Name). Msg("policy does not contain required output permission section") return ErrNoOutputPerms } @@ -86,39 +85,12 @@ func (p *Output) prepareElasticsearch( agent.ElasticsearchOutputs[p.Name] = output } - // Now that there is a 'migrateElasticsearchOutputs' on internal/pkg/dl/migration.go, - // the code below likely can be deleted and this function can only rely on the new agent model. - - // Migration path, see https://github.com/elastic/fleet-server/issues/1672: - // - force API keys to be regenerated: - // - make them empty - // - add them to Old API key, so they'll be deleted - // - use agent.ElasticsearchOutputs instead of agent.Default* - if agent.DefaultAPIKey != "" { - output.APIKey = "" - output.APIKeyID = "" - output.PolicyPermissionsHash = agent.PolicyOutputPermissionsHash - output.ToRetireAPIKeys = append(output.ToRetireAPIKeys, - model.ToRetireAPIKeysItems{ - ID: agent.DefaultAPIKeyID, - RetiredAt: time.Now().UTC().Format(time.RFC3339), - }) - output.ToRetireAPIKeys = append(output.ToRetireAPIKeys, - agent.DefaultAPIKeyHistory...) - - agent.DefaultAPIKey = "" - agent.DefaultAPIKeyID = "" - agent.PolicyOutputPermissionsHash = "" - agent.DefaultAPIKeyHistory = nil - } - // Determine whether we need to generate an output ApiKey. - // This is accomplished by comparing the sha2 hash stored in the agent - // record with the precalculated sha2 hash of the role. + // This is accomplished by comparing the sha2 hash stored in the corresponding + // output in the agent record with the precalculated sha2 hash of the role. // Note: This will need to be updated when doing multi-cluster elasticsearch support - // Currently, we only have access to the token for the elasticsearch instance fleet-server - // is monitors. When updating for multiple ES instances we need to tie the token to the output. + // Currently, we assume all ES outputs are the same ES fleet-server is connected to. needNewKey := true switch { case output.APIKey == "": @@ -153,10 +125,10 @@ func (p *Output) prepareElasticsearch( output.PolicyPermissionsHash = p.Role.Sha2 // for the sake of consistency // When a new keys is generated we need to update the Agent record, - // this will need to be updated when multiples Elasticsearch output - // are used. + // this will need to be updated when multiples remote Elasticsearch output + // are supported. zlog.Info(). - Str("fleet.role.hash.sha256", p.Role.Sha2). + Str("fleet.policy.role.hash.sha256", p.Role.Sha2). Str(logger.DefaultOutputAPIKeyID, outputAPIKey.ID). Msg("Updating agent record to pick up default output key.") diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go index 133b9e305..a1fb8a8ef 100644 --- a/internal/pkg/policy/policy_output_test.go +++ b/internal/pkg/policy/policy_output_test.go @@ -83,133 +83,7 @@ func TestPolicyESOutputPrepareNoRole(t *testing.T) { bulker.AssertExpectations(t) } -func TestPolicyOutputESPrepare_oldModel(t *testing.T) { - // TODO: ensure the DefaultAPIKeyHistory is copied to ElasticsearchOutputs[].ToRetireAPIKeys - // TODO: ensure current DefaultAPIKeyID id added to ElasticsearchOutputs[].ToRetireAPIKeys - t.Run("Permission hash == Agent Permission Hash -> force generate the key", func(t *testing.T) { - logger := testlog.SetLogger(t) - - apiKey := bulk.APIKey{ID: "test_id_existing", Key: "existing-key"} - wantAPIKey := bulk.APIKey{ID: "test_id_new", Key: "new-key"} - - hashPerm := "abc123" - output := Output{ - Type: OutputTypeElasticsearch, - Name: "test output", - Role: &RoleT{ - Sha2: hashPerm, - Raw: TestPayload, - }, - } - - bulker := ftesting.NewMockBulk() - bulker.On("Update", - mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(nil).Once() - bulker.On("APIKeyCreate", - mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(&wantAPIKey, nil).Once() //nolint:govet // test case - - policyMap := smap.Map{ - "test output": map[string]interface{}{}, - } - - testAgent := &model.Agent{ - DefaultAPIKey: apiKey.Agent(), - PolicyOutputPermissionsHash: hashPerm, - ElasticsearchOutputs: map[string]*model.PolicyOutput{}, - } - - err := output.Prepare(context.Background(), logger, bulker, testAgent, policyMap) - require.NoError(t, err, "expected prepare to pass") - - key, ok := policyMap.GetMap(output.Name)["api_key"].(string) - gotOutput := testAgent.ElasticsearchOutputs[output.Name] - - require.True(t, ok, "api key not present on policy map") - assert.Equal(t, wantAPIKey.Agent(), key) - - // Migration path: copy old values to new ElasticsearchOutputs field - assert.Equal(t, wantAPIKey.Agent(), gotOutput.APIKey) - assert.Equal(t, wantAPIKey.ID, gotOutput.APIKeyID) - assert.Equal(t, output.Role.Sha2, gotOutput.PolicyPermissionsHash) - - // Migration path: ensure Default* fields are left empty - assert.Empty(t, testAgent.DefaultAPIKey) - assert.Empty(t, testAgent.DefaultAPIKeyID) - assert.Empty(t, testAgent.DefaultAPIKeyHistory) - assert.Empty(t, testAgent.PolicyOutputPermissionsHash) - - bulker.AssertExpectations(t) - }) - - t.Run("Permission hash != Agent Permission Hash need to regenerate the key", func(t *testing.T) { - logger := testlog.SetLogger(t) - bulker := ftesting.NewMockBulk() - - oldAPIKey := bulk.APIKey{ID: "test_id", Key: "EXISTING-KEY"} - wantAPIKey := bulk.APIKey{ID: "abc", Key: "new-key"} - hashPerm := "old-HASH" - - bulker.On("Update", - mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(nil).Once() - bulker.On("APIKeyCreate", - mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(&wantAPIKey, nil).Once() //nolint:govet // test case - - output := Output{ - Type: OutputTypeElasticsearch, - Name: "test output", - Role: &RoleT{ - Sha2: "new-hash", - Raw: TestPayload, - }, - } - - policyMap := smap.Map{ - "test output": map[string]interface{}{}, - } - - testAgent := &model.Agent{ - DefaultAPIKey: oldAPIKey.Agent(), - PolicyOutputPermissionsHash: hashPerm, - ElasticsearchOutputs: map[string]*model.PolicyOutput{ - // output.Name: { - // ESDocument: model.ESDocument{}, - // APIKey: oldAPIKey.Agent(), - // ToRetireAPIKeys: nil, - // APIKeyID: oldAPIKey.ID, - // PolicyPermissionsHash: hashPerm, - // }, - }, - } - - err := output.Prepare(context.Background(), logger, bulker, testAgent, policyMap) - require.NoError(t, err, "expected prepare to pass") - - key, ok := policyMap.GetMap(output.Name)["api_key"].(string) - gotOutput := testAgent.ElasticsearchOutputs[output.Name] - - require.True(t, ok, "unable to case api key") - require.Equal(t, wantAPIKey.Agent(), key) - - // Migration path: copy old values to new ElasticsearchOutputs field - assert.Equal(t, wantAPIKey.Agent(), gotOutput.APIKey) - assert.Equal(t, wantAPIKey.ID, gotOutput.APIKeyID) - assert.Equal(t, output.Role.Sha2, gotOutput.PolicyPermissionsHash) - - // Migration path: ensure Default* fields are left empty - assert.Empty(t, testAgent.DefaultAPIKey) - assert.Empty(t, testAgent.DefaultAPIKeyID) - assert.Empty(t, testAgent.DefaultAPIKeyHistory) - assert.Empty(t, testAgent.PolicyOutputPermissionsHash) - - bulker.AssertExpectations(t) - }) -} - -func TestPolicyOutputESPrepare_newModel(t *testing.T) { +func TestPolicyOutputESPrepare(t *testing.T) { t.Run("Permission hash == Agent Permission Hash no need to regenerate the key", func(t *testing.T) { logger := testlog.SetLogger(t) bulker := ftesting.NewMockBulk() From 1fa74ec90451a5554f2aec69ed86588a711383f4 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 1 Aug 2022 10:57:16 +0200 Subject: [PATCH 54/89] rename ElasticSearchOutputs to Outputs and add a 'type' field to it --- internal/pkg/dl/agent_integration_test.go | 18 ++++++------- internal/pkg/dl/migration.go | 27 ++++++++++--------- internal/pkg/dl/migration_integration_test.go | 16 ++++++----- internal/pkg/model/schema.go | 9 ++++--- internal/pkg/policy/policy_output.go | 27 ++++++++++--------- .../policy/policy_output_integration_test.go | 12 +++++---- internal/pkg/policy/policy_output_test.go | 17 +++++++----- model/schema.json | 17 +++++++++--- 8 files changed, 84 insertions(+), 59 deletions(-) diff --git a/internal/pkg/dl/agent_integration_test.go b/internal/pkg/dl/agent_integration_test.go index 93646b531..3abb21b7b 100644 --- a/internal/pkg/dl/agent_integration_test.go +++ b/internal/pkg/dl/agent_integration_test.go @@ -118,7 +118,7 @@ func TestFindAgent_NewModel(t *testing.T) { policyID := uuid.Must(uuid.NewV4()).String() agentID := uuid.Must(uuid.NewV4()).String() - wantElasticsearchOutputs := map[string]*model.PolicyOutput{ + wantOutputs := map[string]*model.PolicyOutput{ "default": { APIKey: "TestFindNewModelAgent_APIKey", ToRetireAPIKeys: []model.ToRetireAPIKeysItems{ @@ -131,13 +131,13 @@ func TestFindAgent_NewModel(t *testing.T) { }, } body, err := json.Marshal(model.Agent{ - PolicyID: policyID, - Active: true, - LastCheckin: nowStr, - LastCheckinStatus: "", - UpdatedAt: nowStr, - EnrolledAt: nowStr, - ElasticsearchOutputs: wantElasticsearchOutputs, + PolicyID: policyID, + Active: true, + LastCheckin: nowStr, + LastCheckinStatus: "", + UpdatedAt: nowStr, + EnrolledAt: nowStr, + Outputs: wantOutputs, }) require.NoError(t, err) @@ -150,5 +150,5 @@ func TestFindAgent_NewModel(t *testing.T) { require.NoError(t, err) assert.Equal(t, agentID, agent.Id) - assert.Equal(t, wantElasticsearchOutputs, agent.ElasticsearchOutputs) + assert.Equal(t, wantOutputs, agent.Outputs) } diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index dcab89f8f..c50e93bb8 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -40,7 +40,7 @@ type ( ) func Migrate(ctx context.Context, bulker bulk.Bulk) error { - for _, fn := range []migrationBodyFn{migrateAgentMetadata, migrateElasticsearchOutputs} { + for _, fn := range []migrationBodyFn{migrateAgentMetadata, migrateOutputs} { if _, err := migrate(ctx, bulker, fn); err != nil { return err } @@ -167,24 +167,25 @@ func migrateAgentMetadata() (string, []byte, error) { // their zero value and an older version of FleetServer can repopulate them. // However, reverting FleetServer to an older version might cause very issue // this change fixes. -func migrateElasticsearchOutputs() (string, []byte, error) { - const migrationName = "ElasticsearchOutputs" +func migrateOutputs() (string, []byte, error) { + const migrationName = "Outputs" root := dsl.NewRoot() root.Query().Bool().MustNot().Exists("elasticsearch_outputs") painless := ` // set up the new filed -if (ctx._source['elasticsearch_outputs']==null) - {ctx._source['elasticsearch_outputs']=new HashMap();} -if (ctx._source['elasticsearch_outputs']['default']==null) - {ctx._source['elasticsearch_outputs']['default']=new HashMap();} - -// copy old values to new 'elasticsearch_outputs' field -ctx._source['elasticsearch_outputs']['default'].to_retire_api_keys=ctx._source.default_api_key_history; -ctx._source['elasticsearch_outputs']['default'].api_key=ctx._source.default_api_key; -ctx._source['elasticsearch_outputs']['default'].api_key_id=ctx._source.default_api_key_id; -ctx._source['elasticsearch_outputs']['default'].policy_permissions_hash=ctx._source.policy_output_permissions_hash; +if (ctx._source['outputs']==null) + {ctx._source['outputs']=new HashMap();} +if (ctx._source['outputs']['default']==null) + {ctx._source['outputs']['default']=new HashMap();} + +// copy old values to new 'outputs' field +ctx._source['outputs']['default'].type="elasticsearch"; +ctx._source['outputs']['default'].to_retire_api_keys=ctx._source.default_api_key_history; +ctx._source['outputs']['default'].api_key=ctx._source.default_api_key; +ctx._source['outputs']['default'].api_key_id=ctx._source.default_api_key_id; +ctx._source['outputs']['default'].policy_permissions_hash=ctx._source.policy_output_permissions_hash; // Erase deprecated fields ctx._source.default_api_key_history=null; diff --git a/internal/pkg/dl/migration_integration_test.go b/internal/pkg/dl/migration_integration_test.go index 8e7b996a9..2ce5282af 100644 --- a/internal/pkg/dl/migration_integration_test.go +++ b/internal/pkg/dl/migration_integration_test.go @@ -66,7 +66,7 @@ func createSomeAgents(t *testing.T, n int, apiKey bulk.APIKey, index string, bul return createdAgents } -func TestMigrateElasticsearchOutputs(t *testing.T) { +func TestMigrateOutputs(t *testing.T) { index, bulker := ftesting.SetupCleanIndex(context.Background(), t, FleetAgents) apiKey := bulk.APIKey{ ID: fmt.Sprint("testAgent_"), @@ -75,12 +75,13 @@ func TestMigrateElasticsearchOutputs(t *testing.T) { agentIDs := createSomeAgents(t, 10, apiKey, index, bulker) - migratedAgents, err := migrate(context.Background(), bulker, migrateElasticsearchOutputs) + migratedAgents, err := migrate(context.Background(), bulker, migrateOutputs) require.NoError(t, err) assert.Equal(t, len(agentIDs), migratedAgents) for i, id := range agentIDs { + wantOutputType := "elasticsearch" wantAPIKey := bulk.APIKey{ ID: fmt.Sprint(apiKey.ID, i), Key: fmt.Sprint(apiKey.Key, i), @@ -94,13 +95,14 @@ func TestMigrateElasticsearchOutputs(t *testing.T) { } // Assert new fields - require.Len(t, got.ElasticsearchOutputs, 1) - assert.Equal(t, wantAPIKey.Agent(), got.ElasticsearchOutputs["default"].APIKey) - assert.Equal(t, wantAPIKey.ID, got.ElasticsearchOutputs["default"].APIKeyID) - assert.Equal(t, wantAPIKey.Agent(), got.ElasticsearchOutputs["default"].APIKey) + require.Len(t, got.Outputs, 1) + assert.Equal(t, wantAPIKey.Agent(), got.Outputs["default"].APIKey) + assert.Equal(t, wantAPIKey.ID, got.Outputs["default"].APIKeyID) + assert.Equal(t, wantAPIKey.Agent(), got.Outputs["default"].APIKey) + assert.Equal(t, wantOutputType, got.Outputs["default"].Type) assert.Equal(t, fmt.Sprint("a_output_permission_SHA_", i), - got.ElasticsearchOutputs["default"].PolicyPermissionsHash) + got.Outputs["default"].PolicyPermissionsHash) // Assert deprecated fields assert.Empty(t, got.DefaultAPIKey) diff --git a/internal/pkg/model/schema.go b/internal/pkg/model/schema.go index 5b12479a5..b61d47935 100644 --- a/internal/pkg/model/schema.go +++ b/internal/pkg/model/schema.go @@ -129,9 +129,6 @@ type Agent struct { // Deprecated. Use Outputs instead. ID of the API key the Elastic Agent uses to authenticate with elasticsearch DefaultAPIKeyID string `json:"default_api_key_id,omitempty"` - // ElasticsearchOutputs is the policy output data for each Elasticsearch output. It maps the output name to its data - ElasticsearchOutputs map[string]*PolicyOutput `json:"elasticsearch_outputs,omitempty"` - // Date/time the Elastic Agent enrolled EnrolledAt string `json:"enrolled_at"` @@ -147,6 +144,9 @@ type Agent struct { // Local metadata information for the Elastic Agent LocalMetadata json.RawMessage `json:"local_metadata,omitempty"` + // Outputs is the policy output data, mapping the output name to its data + Outputs map[string]*PolicyOutput `json:"outputs,omitempty"` + // Packages array Packages []string `json:"packages,omitempty"` @@ -337,6 +337,9 @@ type PolicyOutput struct { // API keys to be invalidated on next agent ack ToRetireAPIKeys []ToRetireAPIKeysItems `json:"to_retire_api_keys,omitempty"` + + // Type is the output type. Currently only Elasticsearch is supported. + Type string `json:"type"` } // Server A Fleet Server diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index 2a29bef1d..dae75331d 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -74,15 +74,15 @@ func (p *Output) prepareElasticsearch( return ErrNoOutputPerms } - output, ok := agent.ElasticsearchOutputs[p.Name] + output, ok := agent.Outputs[p.Name] if !ok { - if agent.ElasticsearchOutputs == nil { - agent.ElasticsearchOutputs = map[string]*model.PolicyOutput{} + if agent.Outputs == nil { + agent.Outputs = map[string]*model.PolicyOutput{} } - zlog.Debug().Msgf("creating agent.ElasticsearchOutputs[%s]", p.Name) + zlog.Debug().Msgf("creating agent.Outputs[%s]", p.Name) output = &model.PolicyOutput{} - agent.ElasticsearchOutputs[p.Name] = output + agent.Outputs[p.Name] = output } // Determine whether we need to generate an output ApiKey. @@ -120,6 +120,7 @@ func (p *Output) prepareElasticsearch( return fmt.Errorf("failed generate output API key: %w", err) } + output.Type = OutputTypeElasticsearch output.APIKey = outputAPIKey.Agent() output.APIKeyID = outputAPIKey.ID output.PolicyPermissionsHash = p.Role.Sha2 // for the sake of consistency @@ -177,10 +178,10 @@ func renderUpdatePainlessScript(outputName string, fields map[string]interface{} // prepare agent.elasticsearch_outputs[OUTPUT_NAME] source.WriteString(fmt.Sprintf(` -if (ctx._source['elasticsearch_outputs']==null) - {ctx._source['elasticsearch_outputs']=new HashMap();} -if (ctx._source['elasticsearch_outputs']['%s']==null) - {ctx._source['elasticsearch_outputs']['%s']=new HashMap();} +if (ctx._source['outputs']==null) + {ctx._source['outputs']=new HashMap();} +if (ctx._source['outputs']['%s']==null) + {ctx._source['outputs']['%s']=new HashMap();} `, outputName, outputName)) for field := range fields { @@ -189,14 +190,14 @@ if (ctx._source['elasticsearch_outputs']['%s']==null) // It's an array that gets deleted when the keys are invalidated. // Thus, append the old API key ID, create the field if necessary. source.WriteString(fmt.Sprintf(` -if (ctx._source['elasticsearch_outputs']['%s'].%s==null) - {ctx._source['elasticsearch_outputs']['%s'].%s=new ArrayList();} -ctx._source['elasticsearch_outputs']['%s'].%s.add(params.%s); +if (ctx._source['outputs']['%s'].%s==null) + {ctx._source['outputs']['%s'].%s=new ArrayList();} +ctx._source['outputs']['%s'].%s.add(params.%s); `, outputName, field, outputName, field, outputName, field, field)) } else { // Update the other fields source.WriteString(fmt.Sprintf(` -ctx._source['elasticsearch_outputs']['%s'].%s=params.%s;`, +ctx._source['outputs']['%s'].%s=params.%s;`, outputName, field, field)) } } diff --git a/internal/pkg/policy/policy_output_integration_test.go b/internal/pkg/policy/policy_output_integration_test.go index 839c435ec..851cd4111 100644 --- a/internal/pkg/policy/policy_output_integration_test.go +++ b/internal/pkg/policy/policy_output_integration_test.go @@ -53,11 +53,12 @@ func TestRenderUpdatePainlessScript(t *testing.T) { Key: "old_" + outputAPIKey.Key, } - wantElasticsearchOutputs := map[string]*model.PolicyOutput{ + wantOutputs := map[string]*model.PolicyOutput{ outputName: { APIKey: outputAPIKey.Agent(), APIKeyID: outputAPIKey.ID, PolicyPermissionsHash: outputPermissionSha, + Type: OutputTypeElasticsearch, ToRetireAPIKeys: append(tt.existingToRetireAPIKeys, model.ToRetireAPIKeysItems{ ID: previousAPIKey.ID, RetiredAt: nowStr}), @@ -71,8 +72,9 @@ func TestRenderUpdatePainlessScript(t *testing.T) { LastCheckinStatus: "", UpdatedAt: nowStr, EnrolledAt: nowStr, - ElasticsearchOutputs: map[string]*model.PolicyOutput{ + Outputs: map[string]*model.PolicyOutput{ outputName: { + Type: OutputTypeElasticsearch, APIKey: previousAPIKey.Agent(), APIKeyID: previousAPIKey.ID, PolicyPermissionsHash: "old_" + outputPermissionSha, @@ -80,7 +82,7 @@ func TestRenderUpdatePainlessScript(t *testing.T) { }, } if tt.existingToRetireAPIKeys != nil { - agentModel.ElasticsearchOutputs[outputName].ToRetireAPIKeys = + agentModel.Outputs[outputName].ToRetireAPIKeys = tt.existingToRetireAPIKeys } @@ -114,8 +116,8 @@ func TestRenderUpdatePainlessScript(t *testing.T) { require.NoError(t, err) assert.Equal(t, agentID, gotAgent.Id) - assert.Len(t, gotAgent.ElasticsearchOutputs, len(wantElasticsearchOutputs)) - assert.Equal(t, wantElasticsearchOutputs, gotAgent.ElasticsearchOutputs) + assert.Len(t, gotAgent.Outputs, len(wantOutputs)) + assert.Equal(t, wantOutputs, gotAgent.Outputs) }) } } diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go index a1fb8a8ef..d3e7e18c2 100644 --- a/internal/pkg/policy/policy_output_test.go +++ b/internal/pkg/policy/policy_output_test.go @@ -105,13 +105,14 @@ func TestPolicyOutputESPrepare(t *testing.T) { } testAgent := &model.Agent{ - ElasticsearchOutputs: map[string]*model.PolicyOutput{ + Outputs: map[string]*model.PolicyOutput{ output.Name: { ESDocument: model.ESDocument{}, APIKey: apiKey.Agent(), ToRetireAPIKeys: nil, APIKeyID: apiKey.ID, PolicyPermissionsHash: hashPerm, + Type: OutputTypeElasticsearch, }, }, } @@ -120,7 +121,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { require.NoError(t, err, "expected prepare to pass") key, ok := policyMap.GetMap(output.Name)["api_key"].(string) - gotOutput := testAgent.ElasticsearchOutputs[output.Name] + gotOutput := testAgent.Outputs[output.Name] require.True(t, ok, "api key not present on policy map") assert.Equal(t, apiKey.Agent(), key) @@ -128,6 +129,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { assert.Equal(t, apiKey.Agent(), gotOutput.APIKey) assert.Equal(t, apiKey.ID, gotOutput.APIKeyID) assert.Equal(t, output.Role.Sha2, gotOutput.PolicyPermissionsHash) + assert.Equal(t, output.Type, gotOutput.Type) assert.Empty(t, gotOutput.ToRetireAPIKeys) // Old model must always remain empty @@ -172,13 +174,14 @@ func TestPolicyOutputESPrepare(t *testing.T) { } testAgent := &model.Agent{ - ElasticsearchOutputs: map[string]*model.PolicyOutput{ + Outputs: map[string]*model.PolicyOutput{ output.Name: { ESDocument: model.ESDocument{}, APIKey: oldAPIKey.Agent(), ToRetireAPIKeys: nil, APIKeyID: oldAPIKey.ID, PolicyPermissionsHash: hashPerm, + Type: OutputTypeElasticsearch, }, }, } @@ -187,7 +190,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { require.NoError(t, err, "expected prepare to pass") key, ok := policyMap.GetMap(output.Name)["api_key"].(string) - gotOutput := testAgent.ElasticsearchOutputs[output.Name] + gotOutput := testAgent.Outputs[output.Name] require.True(t, ok, "unable to case api key") require.Equal(t, wantAPIKey.Agent(), key) @@ -195,6 +198,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { assert.Equal(t, wantAPIKey.Agent(), gotOutput.APIKey) assert.Equal(t, wantAPIKey.ID, gotOutput.APIKeyID) assert.Equal(t, output.Role.Sha2, gotOutput.PolicyPermissionsHash) + assert.Equal(t, output.Type, gotOutput.Type) // assert.Contains(t, gotOutput.ToRetireAPIKeys, oldAPIKey.ID) // TODO: assert on bulker.Update @@ -231,13 +235,13 @@ func TestPolicyOutputESPrepare(t *testing.T) { "test output": map[string]interface{}{}, } - testAgent := &model.Agent{ElasticsearchOutputs: map[string]*model.PolicyOutput{}} + testAgent := &model.Agent{Outputs: map[string]*model.PolicyOutput{}} err := output.Prepare(context.Background(), logger, bulker, testAgent, policyMap) require.NoError(t, err, "expected prepare to pass") key, ok := policyMap.GetMap(output.Name)["api_key"].(string) - gotOutput := testAgent.ElasticsearchOutputs[output.Name] + gotOutput := testAgent.Outputs[output.Name] require.True(t, ok, "unable to case api key") assert.Equal(t, apiKey.Agent(), key) @@ -245,6 +249,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { assert.Equal(t, apiKey.Agent(), gotOutput.APIKey) assert.Equal(t, apiKey.ID, gotOutput.APIKeyID) assert.Equal(t, output.Role.Sha2, gotOutput.PolicyPermissionsHash) + assert.Equal(t, output.Type, gotOutput.Type) assert.Empty(t, gotOutput.ToRetireAPIKeys) // Old model must always remain empty diff --git a/model/schema.json b/model/schema.json index 88aecc635..0cac892bd 100644 --- a/model/schema.json +++ b/model/schema.json @@ -244,6 +244,7 @@ "name" ] }, + "server-metadata": { "title": "Server Metadata", "description": "A Fleet Server metadata", @@ -264,6 +265,7 @@ "version" ] }, + "server": { "title": "Server", "description": "A Fleet Server", @@ -284,6 +286,7 @@ "server" ] }, + "policy": { "title": "Policy", "description": "A policy that an Elastic Agent is attached to", @@ -329,6 +332,7 @@ "default_fleet_server" ] }, + "policy-leader": { "title": "Policy Leader", "description": "The current leader Fleet Server for a policy", @@ -384,13 +388,18 @@ "policy_permissions_hash": { "description": "The policy output permissions hash", "type": "string" + }, + "type": { + "description": "Type is the output type. Currently only Elasticsearch is supported.", + "type": "string" } }, "required": [ "api_key", "api_key_history", "api_key_id", - "policy_permissions_hash" + "policy_permissions_hash", + "type" ] }, @@ -515,8 +524,8 @@ "description": "Deprecated. Use Outputs instead. Default API Key History", "$ref": "#/definitions/to_retire_api_keys" }, - "elasticsearch_outputs": { - "description": "ElasticsearchOutputs is the policy output data for each Elasticsearch output. It maps the output name to its data", + "outputs": { + "description": "Outputs is the policy output data, mapping the output name to its data", "type": "object", "additionalProperties": { "$ref": "#/definitions/policy_output"} }, @@ -548,6 +557,7 @@ "status" ] }, + "enrollment_api_key": { "title": "Enrollment API key", "description": "An Elastic Agent enrollment API key", @@ -591,6 +601,7 @@ ] } }, + "checkin": { "title": "Checkin", "description": "An Elastic Agent checkin to Fleet", From 17787e1367899758e90d1147162cf141cb069216 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 1 Aug 2022 11:49:33 +0200 Subject: [PATCH 55/89] . --- internal/pkg/api/handleAck.go | 8 +-- internal/pkg/api/handleCheckin.go | 1 - internal/pkg/dl/agent_integration_test.go | 2 + internal/pkg/dl/migration.go | 54 +++++++++++++++++-- internal/pkg/dl/migration_integration_test.go | 2 +- 5 files changed, 57 insertions(+), 10 deletions(-) diff --git a/internal/pkg/api/handleAck.go b/internal/pkg/api/handleAck.go index 3f284b5da..15924d054 100644 --- a/internal/pkg/api/handleAck.go +++ b/internal/pkg/api/handleAck.go @@ -15,6 +15,8 @@ import ( "strings" "time" + "github.com/pkg/errors" + "github.com/elastic/fleet-server/v7/internal/pkg/bulk" "github.com/elastic/fleet-server/v7/internal/pkg/cache" "github.com/elastic/fleet-server/v7/internal/pkg/config" @@ -24,7 +26,6 @@ import ( "github.com/elastic/fleet-server/v7/internal/pkg/logger" "github.com/elastic/fleet-server/v7/internal/pkg/model" "github.com/elastic/fleet-server/v7/internal/pkg/policy" - "github.com/pkg/errors" "github.com/julienschmidt/httprouter" "github.com/rs/zerolog" @@ -337,8 +338,9 @@ func (ack *AckT) handlePolicyChange(ctx context.Context, zlog zerolog.Logger, ag Int64("rev.coordinatorIdx", rev.CoordinatorIdx). Msg("ack policy revision") - if ok && rev.PolicyID == agent.PolicyID && (rev.RevisionIdx > currRev || - (rev.RevisionIdx == currRev && rev.CoordinatorIdx > currCoord)) { + if ok && rev.PolicyID == agent.PolicyID && + (rev.RevisionIdx > currRev || + (rev.RevisionIdx == currRev && rev.CoordinatorIdx > currCoord)) { found = true currRev = rev.RevisionIdx currCoord = rev.CoordinatorIdx diff --git a/internal/pkg/api/handleCheckin.go b/internal/pkg/api/handleCheckin.go index 2d38cb67b..2752dd147 100644 --- a/internal/pkg/api/handleCheckin.go +++ b/internal/pkg/api/handleCheckin.go @@ -61,7 +61,6 @@ func (rt Router) handleCheckin(w http.ResponseWriter, r *http.Request, ps httpro Logger() err := rt.ct.handleCheckin(&zlog, w, r, id) - if err != nil { cntCheckin.IncError(err) resp := NewHTTPErrResp(err) diff --git a/internal/pkg/dl/agent_integration_test.go b/internal/pkg/dl/agent_integration_test.go index 3abb21b7b..a53b327fc 100644 --- a/internal/pkg/dl/agent_integration_test.go +++ b/internal/pkg/dl/agent_integration_test.go @@ -19,6 +19,7 @@ import ( "github.com/elastic/fleet-server/v7/internal/pkg/bulk" "github.com/elastic/fleet-server/v7/internal/pkg/model" + "github.com/elastic/fleet-server/v7/internal/pkg/policy" ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing" ) @@ -120,6 +121,7 @@ func TestFindAgent_NewModel(t *testing.T) { wantOutputs := map[string]*model.PolicyOutput{ "default": { + Type: policy.OutputTypeElasticsearch, APIKey: "TestFindNewModelAgent_APIKey", ToRetireAPIKeys: []model.ToRetireAPIKeysItems{ { diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index c50e93bb8..220abf402 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -14,6 +14,7 @@ import ( "github.com/elastic/fleet-server/v7/internal/pkg/bulk" "github.com/elastic/fleet-server/v7/internal/pkg/dsl" + "github.com/elastic/fleet-server/v7/internal/pkg/policy" "github.com/elastic/go-elasticsearch/v7/esapi" "github.com/pkg/errors" @@ -40,7 +41,7 @@ type ( ) func Migrate(ctx context.Context, bulker bulk.Bulk) error { - for _, fn := range []migrationBodyFn{migrateAgentMetadata, migrateOutputs} { + for _, fn := range []migrationBodyFn{migrateAgentMetadata, migrateAgentOutputs} { if _, err := migrate(ctx, bulker, fn); err != nil { return err } @@ -157,18 +158,61 @@ func migrateAgentMetadata() (string, []byte, error) { return migrationName, body, nil } +// migrateAgentOutputs performs the necessary changes on the Agent documents +// to introduce the `Outputs` field. +// // FleetServer 8.4.0 introduces a new field to the Agent document, Outputs, to // store the outputs credentials and data. The DefaultAPIKey, DefaultAPIKeyID, // DefaultAPIKeyHistory and PolicyOutputPermissionsHash are now deprecated in // favour of the new `Outputs` fields, which maps the output name to its data. // This change fixes https://github.com/elastic/fleet-server/issues/1672. - +// // The change is backward compatible as the deprecated fields are just set to // their zero value and an older version of FleetServer can repopulate them. // However, reverting FleetServer to an older version might cause very issue // this change fixes. -func migrateOutputs() (string, []byte, error) { - const migrationName = "Outputs" +func migrateAgentOutputs() (string, []byte, error) { + const migrationName = "AgentOutputs" + + root := dsl.NewRoot() + root.Query().Bool().MustNot().Exists("elasticsearch_outputs") + + painless := ` +// set up the new filed +if (ctx._source['outputs']==null) + {ctx._source['outputs']=new HashMap();} +if (ctx._source['outputs']['default']==null) + {ctx._source['outputs']['default']=new HashMap();} + +// copy old values to new 'outputs' field +ctx._source['outputs']['default'].type="` + policy.OutputTypeElasticsearch + `"; +ctx._source['outputs']['default'].to_retire_api_keys=ctx._source.default_api_key_history; +ctx._source['outputs']['default'].api_key=ctx._source.default_api_key; +ctx._source['outputs']['default'].api_key_id=ctx._source.default_api_key_id; +ctx._source['outputs']['default'].policy_permissions_hash=ctx._source.policy_output_permissions_hash; + +// Erase deprecated fields +ctx._source.default_api_key_history=null; +ctx._source.default_api_key=""; +ctx._source.default_api_key_id=""; +ctx._source.policy_output_permissions_hash=""; +` + root.Param("script", painless) + + body, err := root.MarshalJSON() + if err != nil { + return migrationName, nil, fmt.Errorf("could not marshal ES query: %w", err) + } + + return migrationName, body, nil +} + +// migratePolicyCoordinatorIdx increases the policy's CoordinatorIdx to force +// a policy update ensuring the output data will be migrated to the new +// Agent.Outputs field. See migrateAgentOutputs and https://github.com/elastic/fleet-server/issues/1672 +// for details. +func migratePolicyCoordinatorIdx() (string, []byte, error) { + const migrationName = "AgentOutputs" root := dsl.NewRoot() root.Query().Bool().MustNot().Exists("elasticsearch_outputs") @@ -181,7 +225,7 @@ if (ctx._source['outputs']['default']==null) {ctx._source['outputs']['default']=new HashMap();} // copy old values to new 'outputs' field -ctx._source['outputs']['default'].type="elasticsearch"; +ctx._source['outputs']['default'].type="` + policy.OutputTypeElasticsearch + `"; ctx._source['outputs']['default'].to_retire_api_keys=ctx._source.default_api_key_history; ctx._source['outputs']['default'].api_key=ctx._source.default_api_key; ctx._source['outputs']['default'].api_key_id=ctx._source.default_api_key_id; diff --git a/internal/pkg/dl/migration_integration_test.go b/internal/pkg/dl/migration_integration_test.go index 2ce5282af..a8fc4e0e2 100644 --- a/internal/pkg/dl/migration_integration_test.go +++ b/internal/pkg/dl/migration_integration_test.go @@ -75,7 +75,7 @@ func TestMigrateOutputs(t *testing.T) { agentIDs := createSomeAgents(t, 10, apiKey, index, bulker) - migratedAgents, err := migrate(context.Background(), bulker, migrateOutputs) + migratedAgents, err := migrate(context.Background(), bulker, migrateAgentOutputs) require.NoError(t, err) assert.Equal(t, len(agentIDs), migratedAgents) From f2e70c51de0ed11c89aee4c847d2e53cadb36879 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 1 Aug 2022 12:17:33 +0200 Subject: [PATCH 56/89] fix import cycle --- internal/pkg/dl/migration.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 220abf402..4f3384b78 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -12,13 +12,12 @@ import ( "net/http" "time" - "github.com/elastic/fleet-server/v7/internal/pkg/bulk" - "github.com/elastic/fleet-server/v7/internal/pkg/dsl" - "github.com/elastic/fleet-server/v7/internal/pkg/policy" - "github.com/elastic/go-elasticsearch/v7/esapi" "github.com/pkg/errors" "github.com/rs/zerolog/log" + + "github.com/elastic/fleet-server/v7/internal/pkg/bulk" + "github.com/elastic/fleet-server/v7/internal/pkg/dsl" ) type ( @@ -185,7 +184,7 @@ if (ctx._source['outputs']['default']==null) {ctx._source['outputs']['default']=new HashMap();} // copy old values to new 'outputs' field -ctx._source['outputs']['default'].type="` + policy.OutputTypeElasticsearch + `"; +ctx._source['outputs']['default'].type="elasticsearch"; ctx._source['outputs']['default'].to_retire_api_keys=ctx._source.default_api_key_history; ctx._source['outputs']['default'].api_key=ctx._source.default_api_key; ctx._source['outputs']['default'].api_key_id=ctx._source.default_api_key_id; @@ -225,7 +224,7 @@ if (ctx._source['outputs']['default']==null) {ctx._source['outputs']['default']=new HashMap();} // copy old values to new 'outputs' field -ctx._source['outputs']['default'].type="` + policy.OutputTypeElasticsearch + `"; +ctx._source['outputs']['default'].type="elasticsearch"; ctx._source['outputs']['default'].to_retire_api_keys=ctx._source.default_api_key_history; ctx._source['outputs']['default'].api_key=ctx._source.default_api_key; ctx._source['outputs']['default'].api_key_id=ctx._source.default_api_key_id; From 056d3f07ed5099193f0488742257ddc488fb0ef4 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 1 Aug 2022 12:32:55 +0200 Subject: [PATCH 57/89] adjust handleAck invalidate API keys --- internal/pkg/api/handleAck.go | 34 +++++++++++++++++++---------- internal/pkg/api/handleAck_test.go | 35 +++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/internal/pkg/api/handleAck.go b/internal/pkg/api/handleAck.go index 15924d054..c84cc6e47 100644 --- a/internal/pkg/api/handleAck.go +++ b/internal/pkg/api/handleAck.go @@ -351,17 +351,7 @@ func (ack *AckT) handlePolicyChange(ctx context.Context, zlog zerolog.Logger, ag return nil } - sz := len(agent.DefaultAPIKeyHistory) - if sz > 0 { - ids := make([]string, sz) - for i := 0; i < sz; i++ { - ids[i] = agent.DefaultAPIKeyHistory[i].ID - } - log.Info().Strs("ids", ids).Msg("Invalidate old API keys") - if err := ack.bulk.APIKeyInvalidate(ctx, ids...); err != nil { - log.Info().Err(err).Strs("ids", ids).Msg("Failed to invalidate API keys") - } - } + ack.invalidateAPIKeys(ctx, agent) body := makeUpdatePolicyBody( agent.PolicyID, @@ -387,6 +377,26 @@ func (ack *AckT) handlePolicyChange(ctx context.Context, zlog zerolog.Logger, ag return errors.Wrap(err, "handlePolicyChange update") } +func (ack *AckT) invalidateAPIKeys(ctx context.Context, agent *model.Agent) { + var toRetire []model.ToRetireAPIKeysItems + + for _, out := range agent.Outputs { + toRetire = append(toRetire, out.ToRetireAPIKeys...) + } + + size := len(toRetire) + if size > 0 { + ids := make([]string, size) + for i := 0; i < size; i++ { + ids[i] = toRetire[i].ID + } + log.Info().Strs("fleet.policy.apiKeyIDsToRetire", ids).Msg("Invalidate old API keys") + if err := ack.bulk.APIKeyInvalidate(ctx, ids...); err != nil { + log.Info().Err(err).Strs("ids", ids).Msg("Failed to invalidate API keys") + } + } +} + func (ack *AckT) handleUnenroll(ctx context.Context, zlog zerolog.Logger, agent *model.Agent) error { apiKeys := _getAPIKeyIDs(agent) if len(apiKeys) > 0 { @@ -443,10 +453,12 @@ func (ack *AckT) handleUpgrade(ctx context.Context, zlog zerolog.Logger, agent * } func _getAPIKeyIDs(agent *model.Agent) []string { + keys := make([]string, 0, 1) if agent.AccessAPIKeyID != "" { keys = append(keys, agent.AccessAPIKeyID) } + // TODO: FIX ME if agent.DefaultAPIKeyID != "" { keys = append(keys, agent.DefaultAPIKeyID) } diff --git a/internal/pkg/api/handleAck_test.go b/internal/pkg/api/handleAck_test.go index 90c961456..ece0f58f7 100644 --- a/internal/pkg/api/handleAck_test.go +++ b/internal/pkg/api/handleAck_test.go @@ -15,13 +15,14 @@ import ( "net/http" "testing" + "github.com/google/go-cmp/cmp" + "github.com/elastic/fleet-server/v7/internal/pkg/cache" "github.com/elastic/fleet-server/v7/internal/pkg/config" "github.com/elastic/fleet-server/v7/internal/pkg/es" "github.com/elastic/fleet-server/v7/internal/pkg/model" ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing" testlog "github.com/elastic/fleet-server/v7/internal/pkg/testing/log" - "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -439,3 +440,35 @@ func TestHandleAckEvents(t *testing.T) { }) } } + +func TestInvalidateAPIKeys(t *testing.T) { + toRetire1 := []model.ToRetireAPIKeysItems{{ + ID: "toRetire1", + }} + toRetire2 := []model.ToRetireAPIKeysItems{{ + ID: "toRetire2_0", + }, { + ID: "toRetire2_1", + }} + var toRetire3 []model.ToRetireAPIKeysItems + + want := []string{"toRetire1", "toRetire2_0", "toRetire2_1"} + + agent := model.Agent{ + Outputs: map[string]*model.PolicyOutput{ + "1": {ToRetireAPIKeys: toRetire1}, + "2": {ToRetireAPIKeys: toRetire2}, + "3": {ToRetireAPIKeys: toRetire3}, + }, + } + + bulker := ftesting.NewMockBulk() + bulker.On("APIKeyInvalidate", + context.Background(), want). + Return(nil) + + ack := &AckT{bulk: bulker} + ack.invalidateAPIKeys(context.Background(), &agent) + + bulker.AssertExpectations(t) +} From 7847924c0273b260075ede4a03a89fbfc355e118 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 2 Aug 2022 13:51:32 +0200 Subject: [PATCH 58/89] migrate policy coordinator idx working --- internal/pkg/coordinator/monitor.go | 1 + internal/pkg/dl/agent_integration_test.go | 3 +- internal/pkg/dl/migration.go | 52 +++++----------- internal/pkg/dl/migration_integration_test.go | 62 +++++++++++++++++++ internal/pkg/testing/setup.go | 2 +- licenses/license_header.go | 19 +++++- 6 files changed, 98 insertions(+), 41 deletions(-) diff --git a/internal/pkg/coordinator/monitor.go b/internal/pkg/coordinator/monitor.go index 0899298a6..4a70104b4 100644 --- a/internal/pkg/coordinator/monitor.go +++ b/internal/pkg/coordinator/monitor.go @@ -557,6 +557,7 @@ func getAPIKeyIDs(agent *model.Agent) []string { if agent.AccessAPIKeyID != "" { keys = append(keys, agent.AccessAPIKeyID) } + // TODO: FIX ME if agent.DefaultAPIKeyID != "" { keys = append(keys, agent.DefaultAPIKeyID) } diff --git a/internal/pkg/dl/agent_integration_test.go b/internal/pkg/dl/agent_integration_test.go index a53b327fc..6e883676c 100644 --- a/internal/pkg/dl/agent_integration_test.go +++ b/internal/pkg/dl/agent_integration_test.go @@ -19,7 +19,6 @@ import ( "github.com/elastic/fleet-server/v7/internal/pkg/bulk" "github.com/elastic/fleet-server/v7/internal/pkg/model" - "github.com/elastic/fleet-server/v7/internal/pkg/policy" ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing" ) @@ -121,7 +120,7 @@ func TestFindAgent_NewModel(t *testing.T) { wantOutputs := map[string]*model.PolicyOutput{ "default": { - Type: policy.OutputTypeElasticsearch, + Type: "elasticsearch", APIKey: "TestFindNewModelAgent_APIKey", ToRetireAPIKeys: []model.ToRetireAPIKeysItems{ { diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 4f3384b78..11d920fcd 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -21,7 +21,7 @@ import ( ) type ( - migrationBodyFn func() (string, []byte, error) + migrationBodyFn func() (string, string, []byte, error) migrationResponse struct { Took int `json:"took"` TimedOut bool `json:"timed_out"` @@ -49,7 +49,7 @@ func Migrate(ctx context.Context, bulker bulk.Bulk) error { return nil } -func applyMigration(ctx context.Context, name string, bulker bulk.Bulk, body []byte) (migrationResponse, error) { +func applyMigration(ctx context.Context, name string, index string, bulker bulk.Bulk, body []byte) (migrationResponse, error) { start := time.Now() client := bulker.Client() @@ -63,7 +63,7 @@ func applyMigration(ctx context.Context, name string, bulker bulk.Bulk, body []b client.UpdateByQuery.WithConflicts("proceed"), } - res, err := client.UpdateByQuery([]string{FleetAgents}, opts...) + res, err := client.UpdateByQuery([]string{index}, opts...) if err != nil { return migrationResponse{}, err } @@ -110,12 +110,12 @@ func applyMigration(ctx context.Context, name string, bulker bulk.Bulk, body []b func migrate(ctx context.Context, bulker bulk.Bulk, fn migrationBodyFn) (int, error) { var updatedDocs int for { - name, body, err := fn() + name, index, body, err := fn() if err != nil { return updatedDocs, fmt.Errorf(": %w", err) } - resp, err := applyMigration(ctx, name, bulker, body) + resp, err := applyMigration(ctx, name, index, bulker, body) if err != nil { return updatedDocs, fmt.Errorf("failed to apply migration %q: %w", name, err) @@ -141,7 +141,7 @@ func migrate(ctx context.Context, bulker bulk.Bulk, fn migrationBodyFn) (int, er // can be run in parallel at the same time. // // As the update only occurs once, the 99.9% case is a noop. -func migrateAgentMetadata() (string, []byte, error) { +func migrateAgentMetadata() (string, string, []byte, error) { const migrationName = "AgentMetadata" root := dsl.NewRoot() root.Query().Bool().MustNot().Exists("agent.id") @@ -151,10 +151,10 @@ func migrateAgentMetadata() (string, []byte, error) { body, err := root.MarshalJSON() if err != nil { - return migrationName, nil, fmt.Errorf("could not marshal ES query: %w", err) + return migrationName, FleetAgents, nil, fmt.Errorf("could not marshal ES query: %w", err) } - return migrationName, body, nil + return migrationName, FleetAgents, body, nil } // migrateAgentOutputs performs the necessary changes on the Agent documents @@ -170,7 +170,7 @@ func migrateAgentMetadata() (string, []byte, error) { // their zero value and an older version of FleetServer can repopulate them. // However, reverting FleetServer to an older version might cause very issue // this change fixes. -func migrateAgentOutputs() (string, []byte, error) { +func migrateAgentOutputs() (string, string, []byte, error) { const migrationName = "AgentOutputs" root := dsl.NewRoot() @@ -200,48 +200,30 @@ ctx._source.policy_output_permissions_hash=""; body, err := root.MarshalJSON() if err != nil { - return migrationName, nil, fmt.Errorf("could not marshal ES query: %w", err) + return migrationName, FleetAgents, nil, fmt.Errorf("could not marshal ES query: %w", err) } - return migrationName, body, nil + return migrationName, FleetAgents, body, nil } // migratePolicyCoordinatorIdx increases the policy's CoordinatorIdx to force // a policy update ensuring the output data will be migrated to the new // Agent.Outputs field. See migrateAgentOutputs and https://github.com/elastic/fleet-server/issues/1672 // for details. -func migratePolicyCoordinatorIdx() (string, []byte, error) { - const migrationName = "AgentOutputs" +func migratePolicyCoordinatorIdx() (string, string, []byte, error) { + const migrationName = "PolicyCoordinatorIdx" root := dsl.NewRoot() - root.Query().Bool().MustNot().Exists("elasticsearch_outputs") + root.Query().MatchAll() painless := ` -// set up the new filed -if (ctx._source['outputs']==null) - {ctx._source['outputs']=new HashMap();} -if (ctx._source['outputs']['default']==null) - {ctx._source['outputs']['default']=new HashMap();} - -// copy old values to new 'outputs' field -ctx._source['outputs']['default'].type="elasticsearch"; -ctx._source['outputs']['default'].to_retire_api_keys=ctx._source.default_api_key_history; -ctx._source['outputs']['default'].api_key=ctx._source.default_api_key; -ctx._source['outputs']['default'].api_key_id=ctx._source.default_api_key_id; -ctx._source['outputs']['default'].policy_permissions_hash=ctx._source.policy_output_permissions_hash; - -// Erase deprecated fields -ctx._source.default_api_key_history=null; -ctx._source.default_api_key=""; -ctx._source.default_api_key_id=""; -ctx._source.policy_output_permissions_hash=""; -` + ctx._source.coordinator_idx++;` root.Param("script", painless) body, err := root.MarshalJSON() if err != nil { - return migrationName, nil, fmt.Errorf("could not marshal ES query: %w", err) + return migrationName, FleetPolicies, nil, fmt.Errorf("could not marshal ES query: %w", err) } - return migrationName, body, nil + return migrationName, FleetPolicies, body, nil } diff --git a/internal/pkg/dl/migration_integration_test.go b/internal/pkg/dl/migration_integration_test.go index a8fc4e0e2..414be3209 100644 --- a/internal/pkg/dl/migration_integration_test.go +++ b/internal/pkg/dl/migration_integration_test.go @@ -66,6 +66,68 @@ func createSomeAgents(t *testing.T, n int, apiKey bulk.APIKey, index string, bul return createdAgents } +func createSomePolicies(t *testing.T, n int, index string, bulker bulk.Bulk) []string { + t.Helper() + + var created []string + + for i := 0; i < n; i++ { + now := time.Now().UTC() + nowStr := now.Format(time.RFC3339) + + policyModel := model.Policy{ + ESDocument: model.ESDocument{}, + CoordinatorIdx: int64(i), + Data: nil, + DefaultFleetServer: false, + PolicyID: fmt.Sprint(i), + RevisionIdx: 1, + Timestamp: nowStr, + UnenrollTimeout: 0, + } + + body, err := json.Marshal(policyModel) + require.NoError(t, err) + + policyDocID, err := bulker.Create( + context.Background(), index, "", body, bulk.WithRefresh()) + require.NoError(t, err) + + created = append(created, policyDocID) + } + + return created +} + +func TestPolicyCoordinatorIdx(t *testing.T) { + index, bulker := ftesting.SetupCleanIndex(context.Background(), t, FleetPolicies) + + docIDs := createSomePolicies(t, 25, index, bulker) + + migrated, err := migrate(context.Background(), bulker, migratePolicyCoordinatorIdx) + require.NoError(t, err) + + require.Equal(t, len(docIDs), migrated) + + for i := range docIDs { + policies, err := QueryLatestPolicies( + context.Background(), bulker, WithIndexName(index)) + if err != nil { + assert.NoError(t, err, "failed to query latest policies") // we want to continue even if a single agent fails + continue + } + + var got model.Policy + for _, p := range policies { + if p.PolicyID == fmt.Sprint(i) { + got = p + } + } + + assert.Equal(t, int64(i+1), got.CoordinatorIdx) + } +} + func TestMigrateOutputs(t *testing.T) { index, bulker := ftesting.SetupCleanIndex(context.Background(), t, FleetAgents) apiKey := bulk.APIKey{ diff --git a/internal/pkg/testing/setup.go b/internal/pkg/testing/setup.go index d7a1aa27d..d5ca6c994 100644 --- a/internal/pkg/testing/setup.go +++ b/internal/pkg/testing/setup.go @@ -39,7 +39,7 @@ fleet: func init() { os.Setenv("ELASTICSEARCH_SERVICE_TOKEN", - "AAEAAWVsYXN0aWMvZmxlZXQtc2VydmVyL3Rva2VuMToyNzJ5dmhndFM3UzVsb1h3SERhT0dB") + "AAEAAWVsYXN0aWMvZmxlZXQtc2VydmVyL3Rva2VuMTozUTBwOHVFWVRYT3Y0dXZnRXktV29n") os.Setenv("ELASTICSEARCH_HOSTS", "localhost:9200") os.Setenv("ELASTICSEARCH_USERNAME", "elastic") os.Setenv("ELASTICSEARCH_PASSWORD", "changeme") diff --git a/licenses/license_header.go b/licenses/license_header.go index 21a8501aa..fc5125bb2 100644 --- a/licenses/license_header.go +++ b/licenses/license_header.go @@ -1,6 +1,19 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. // Code generated by beats/dev-tools/cmd/license/license_generate.go - DO NOT EDIT. From 5f2ff237d4390890cea491fe2d31cff07d56364b Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 2 Aug 2022 14:11:52 +0200 Subject: [PATCH 59/89] adjust migrations: group in migration fn per version --- cmd/fleet/main.go | 10 +++++----- internal/pkg/dl/migration.go | 36 ++++++++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/cmd/fleet/main.go b/cmd/fleet/main.go index 2838896a7..c7634d2e7 100644 --- a/cmd/fleet/main.go +++ b/cmd/fleet/main.go @@ -804,15 +804,16 @@ func (f *FleetServer) runSubsystems(ctx context.Context, cfg *config.Config, g * remoteVersion, err := ver.CheckCompatibility(ctx, esCli, f.bi.Version) if err != nil { if len(remoteVersion) != 0 { - return fmt.Errorf("failed version compatibility check with elasticsearch (Agent: %s, Elasticsearch: %s): %w", f.bi.Version, remoteVersion, err) + return fmt.Errorf("failed version compatibility check with elasticsearch (Agent: %s, Elasticsearch: %s): %w", + f.bi.Version, remoteVersion, err) } return fmt.Errorf("failed version compatibility check with elasticsearch: %w", err) } - // Run migrations; currently it's safe to do it in the background. That may change in the future. - g.Go(loggedRunFunc(ctx, "Migrations", func(ctx context.Context) error { + // Run migrations + loggedRunFunc(ctx, "Migrations", func(ctx context.Context) error { return dl.Migrate(ctx, bulker) - })) + }) // Run scheduler for periodic GC/cleanup gcCfg := cfg.Inputs[0].Server.GC @@ -831,7 +832,6 @@ func (f *FleetServer) runSubsystems(ctx context.Context, cfg *config.Config, g * } // Coordinator policy monitor - // TODO(Anderson): perhapes here to increase the coordinator index pim, err := monitor.New(dl.FleetPolicies, esCli, monCli, monitor.WithFetchSize(cfg.Inputs[0].Monitor.FetchSize), monitor.WithPollTimeout(cfg.Inputs[0].Monitor.PollTimeout), diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 11d920fcd..0f7e41a1f 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -21,6 +21,7 @@ import ( ) type ( + migrationFn func(context.Context, bulk.Bulk) error migrationBodyFn func() (string, string, []byte, error) migrationResponse struct { Took int `json:"took"` @@ -39,9 +40,12 @@ type ( } ) +// Migrate applies, in sequence, the migration functions. Currently, each migration +// function is responsible to ensure it only applies the migration if needed, +// being a no-op otherwise. func Migrate(ctx context.Context, bulker bulk.Bulk) error { - for _, fn := range []migrationBodyFn{migrateAgentMetadata, migrateAgentOutputs} { - if _, err := migrate(ctx, bulker, fn); err != nil { + for _, fn := range []migrationFn{migrateTov7_15, migrateToV8_4} { + if err := fn(ctx, bulker); err != nil { return err } } @@ -49,6 +53,34 @@ func Migrate(ctx context.Context, bulker bulk.Bulk) error { return nil } +func migrateTov7_15(ctx context.Context, bulker bulk.Bulk) error { + _, err := migrate(ctx, bulker, migrateAgentMetadata) + if err != nil { + return fmt.Errorf("v7.15.0 data migration failed: %w", err) + } + + return nil +} + +func migrateToV8_4(ctx context.Context, bulker bulk.Bulk) error { + migrated, err := migrate(ctx, bulker, migrateAgentOutputs) + if err != nil { + return fmt.Errorf("v8.4.0 data migration failed: %w", err) + } + + // The migration was necessary and indeed run, thus we need to regenerate + // the API keys for all agents. In order to do so, we increase the policy + // coordinator index to force a policy update. + if migrated > 0 { + _, err := migrate(ctx, bulker, migratePolicyCoordinatorIdx) + if err != nil { + return fmt.Errorf("v8.4.0 data migration failed: %w", err) + } + } + + return nil +} + func applyMigration(ctx context.Context, name string, index string, bulker bulk.Bulk, body []byte) (migrationResponse, error) { start := time.Now() From ebc21316a2ccb923e8d9ff0a62dd70f1f97d9642 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 2 Aug 2022 14:50:14 +0200 Subject: [PATCH 60/89] add APIKeyIDs to model.Agent --- internal/pkg/model/ext.go | 28 +++++++++++++++-- internal/pkg/model/ext_test.go | 56 ++++++++++++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/internal/pkg/model/ext.go b/internal/pkg/model/ext.go index d89787855..025415cea 100644 --- a/internal/pkg/model/ext.go +++ b/internal/pkg/model/ext.go @@ -27,14 +27,36 @@ func (m *Server) SetTime(t time.Time) { } // CheckDifferentVersion returns Agent version if it is different from ver, otherwise return empty string -func (m *Agent) CheckDifferentVersion(ver string) string { - if m == nil { +func (a *Agent) CheckDifferentVersion(ver string) string { + if a == nil { return "" } - if m.Agent == nil || ver != m.Agent.Version { + if a.Agent == nil || ver != a.Agent.Version { return ver } return "" } + +// APIKeyIDs returns all the API keys, the valid, in-use as well as the one +// marked to be retired. +func (a *Agent) APIKeyIDs() []string { + if a == nil { + return nil + } + keys := make([]string, 0, len(a.Outputs)+1) + if a.AccessAPIKeyID != "" { + keys = append(keys, a.AccessAPIKeyID) + } + + for _, output := range a.Outputs { + keys = append(keys, output.APIKeyID) + for _, key := range output.ToRetireAPIKeys { + keys = append(keys, key.ID) + } + } + + return keys + +} diff --git a/internal/pkg/model/ext_test.go b/internal/pkg/model/ext_test.go index e48194b30..69df04eb5 100644 --- a/internal/pkg/model/ext_test.go +++ b/internal/pkg/model/ext_test.go @@ -2,15 +2,13 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -//go:build !integration -// +build !integration - package model import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" ) func TestAgentGetNewVersion(t *testing.T) { @@ -85,3 +83,55 @@ func TestAgentGetNewVersion(t *testing.T) { }) } } + +func TestAgentAPIKeyIDs(t *testing.T) { + tcs := []struct { + name string + agent Agent + want []string + }{ + { + name: "no API key marked to be retired", + agent: Agent{ + AccessAPIKeyID: "access_api_key_id", + Outputs: map[string]*PolicyOutput{ + "p1": {APIKeyID: "p1_api_key_id"}, + "p2": {APIKeyID: "p2_api_key_id"}, + }, + }, + want: []string{"access_api_key_id", "p1_api_key_id", "p2_api_key_id"}, + }, + { + name: "with API key marked to be retired", + agent: Agent{ + AccessAPIKeyID: "access_api_key_id", + Outputs: map[string]*PolicyOutput{ + "p1": { + APIKeyID: "p1_api_key_id", + ToRetireAPIKeys: []ToRetireAPIKeysItems{{ + ID: "p1_to_retire_key", + }}}, + "p2": { + APIKeyID: "p2_api_key_id", + ToRetireAPIKeys: []ToRetireAPIKeysItems{{ + ID: "p2_to_retire_key", + }}}, + }, + }, + want: []string{ + "access_api_key_id", "p1_api_key_id", "p2_api_key_id", + "p1_to_retire_key", "p2_to_retire_key"}, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + got := tc.agent.APIKeyIDs() + + // if A contains B and B contains A => A = B + assert.Subset(t, tc.want, got) + assert.Subset(t, got, tc.want) + assert.ObjectsAreEqualValues() + }) + } +} From 130b97981e89647fadc2076e6b1718fbdad0c2ff Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 2 Aug 2022 14:50:37 +0200 Subject: [PATCH 61/89] adjust handleAck and monitor to use new agent model --- internal/pkg/api/handleAck.go | 15 +-------------- internal/pkg/coordinator/monitor.go | 18 +++--------------- 2 files changed, 4 insertions(+), 29 deletions(-) diff --git a/internal/pkg/api/handleAck.go b/internal/pkg/api/handleAck.go index c84cc6e47..be1374d2e 100644 --- a/internal/pkg/api/handleAck.go +++ b/internal/pkg/api/handleAck.go @@ -398,7 +398,7 @@ func (ack *AckT) invalidateAPIKeys(ctx context.Context, agent *model.Agent) { } func (ack *AckT) handleUnenroll(ctx context.Context, zlog zerolog.Logger, agent *model.Agent) error { - apiKeys := _getAPIKeyIDs(agent) + apiKeys := agent.APIKeyIDs() if len(apiKeys) > 0 { zlog = zlog.With().Strs(LogAPIKeyID, apiKeys).Logger() @@ -452,19 +452,6 @@ func (ack *AckT) handleUpgrade(ctx context.Context, zlog zerolog.Logger, agent * return nil } -func _getAPIKeyIDs(agent *model.Agent) []string { - - keys := make([]string, 0, 1) - if agent.AccessAPIKeyID != "" { - keys = append(keys, agent.AccessAPIKeyID) - } - // TODO: FIX ME - if agent.DefaultAPIKeyID != "" { - keys = append(keys, agent.DefaultAPIKeyID) - } - return keys -} - // Generate an update script that validates that the policy_id // has not changed underneath us by an upstream process (Kibana or otherwise). // We have a race condition where a user could have assigned a new policy to diff --git a/internal/pkg/coordinator/monitor.go b/internal/pkg/coordinator/monitor.go index 4a70104b4..3990c536b 100644 --- a/internal/pkg/coordinator/monitor.go +++ b/internal/pkg/coordinator/monitor.go @@ -525,11 +525,13 @@ func unenrollAgent(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, a dl.FieldUnenrolledReason: unenrolledReasonTimeout, dl.FieldUpdatedAt: now, } + body, err := fields.Marshal() if err != nil { return err } - apiKeys := getAPIKeyIDs(agent) + + apiKeys := agent.APIKeyIDs() zlog = zlog.With(). Str(logger.AgentID, agent.Id). @@ -552,20 +554,6 @@ func unenrollAgent(ctx context.Context, zlog zerolog.Logger, bulker bulk.Bulk, a return err } -func getAPIKeyIDs(agent *model.Agent) []string { - keys := make([]string, 0, 1) - if agent.AccessAPIKeyID != "" { - keys = append(keys, agent.AccessAPIKeyID) - } - // TODO: FIX ME - if agent.DefaultAPIKeyID != "" { - keys = append(keys, agent.DefaultAPIKeyID) - } - // TODO: should we also collect the old (a.k.a history) api keys to ensure - // they're deleted? - return keys -} - func waitWithContext(ctx context.Context, to time.Duration) error { t := time.NewTimer(to) defer t.Stop() From c39e36572b01ce42375db0f0260d9243213a08bd Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 2 Aug 2022 14:58:24 +0200 Subject: [PATCH 62/89] . --- internal/pkg/model/ext_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/pkg/model/ext_test.go b/internal/pkg/model/ext_test.go index 69df04eb5..5f1667b72 100644 --- a/internal/pkg/model/ext_test.go +++ b/internal/pkg/model/ext_test.go @@ -131,7 +131,6 @@ func TestAgentAPIKeyIDs(t *testing.T) { // if A contains B and B contains A => A = B assert.Subset(t, tc.want, got) assert.Subset(t, got, tc.want) - assert.ObjectsAreEqualValues() }) } } From cd02061eb3ad5ab07471fb3f58ea8a6d05b93e42 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 2 Aug 2022 15:01:59 +0200 Subject: [PATCH 63/89] ops --- internal/pkg/testing/setup.go | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/internal/pkg/testing/setup.go b/internal/pkg/testing/setup.go index d5ca6c994..898afa36b 100644 --- a/internal/pkg/testing/setup.go +++ b/internal/pkg/testing/setup.go @@ -12,7 +12,6 @@ import ( "context" "encoding/json" "errors" - "os" "testing" "github.com/elastic/go-elasticsearch/v7" @@ -38,12 +37,6 @@ fleet: `) func init() { - os.Setenv("ELASTICSEARCH_SERVICE_TOKEN", - "AAEAAWVsYXN0aWMvZmxlZXQtc2VydmVyL3Rva2VuMTozUTBwOHVFWVRYT3Y0dXZnRXktV29n") - os.Setenv("ELASTICSEARCH_HOSTS", "localhost:9200") - os.Setenv("ELASTICSEARCH_USERNAME", "elastic") - os.Setenv("ELASTICSEARCH_PASSWORD", "changeme") - c, err := yaml.NewConfig(defaultCfgData, config.DefaultOptions...) if err != nil { panic(err) @@ -118,18 +111,6 @@ func CleanIndex(ctx context.Context, t *testing.T, bulker bulk.Bulk, index strin cli := bulker.Client() - // // Just running DeleteByQuery with refresh true, does not seem to be enought - // req, err := http.NewRequest(http.MethodPost, index+"/_refresh", nil) - // if err != nil { - // t.Fatalf("could not clean index: failed to create request to refresh index %q: %v", - // index, err) - // } - // _, err = cli.Perform(req) - // if err != nil { - // t.Fatalf("could not clean index: failed to refresh index %q: %v", - // index, err) - // } - res, err := cli.API.DeleteByQuery([]string{index}, bytes.NewReader(query), cli.API.DeleteByQuery.WithContext(ctx), cli.API.DeleteByQuery.WithRefresh(true), From 9a156b59e0d639ea5e29a4a0dc42de9228a83087 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 2 Aug 2022 15:03:03 +0200 Subject: [PATCH 64/89] add/fix file license headers --- internal/pkg/dl/migration_integration_test.go | 4 ++++ internal/pkg/model/schema.go | 4 ++++ .../policy/policy_output_integration_test.go | 4 ++++ licenses/license_header.go | 19 +++---------------- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/internal/pkg/dl/migration_integration_test.go b/internal/pkg/dl/migration_integration_test.go index 414be3209..105ab5055 100644 --- a/internal/pkg/dl/migration_integration_test.go +++ b/internal/pkg/dl/migration_integration_test.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + //go:build integration package dl diff --git a/internal/pkg/model/schema.go b/internal/pkg/model/schema.go index b61d47935..5d71b850e 100644 --- a/internal/pkg/model/schema.go +++ b/internal/pkg/model/schema.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + // Code generated by schema-generate. DO NOT EDIT. package model diff --git a/internal/pkg/policy/policy_output_integration_test.go b/internal/pkg/policy/policy_output_integration_test.go index 851cd4111..03d0cb840 100644 --- a/internal/pkg/policy/policy_output_integration_test.go +++ b/internal/pkg/policy/policy_output_integration_test.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + //go:build integration package policy diff --git a/licenses/license_header.go b/licenses/license_header.go index fc5125bb2..21a8501aa 100644 --- a/licenses/license_header.go +++ b/licenses/license_header.go @@ -1,19 +1,6 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. // Code generated by beats/dev-tools/cmd/license/license_generate.go - DO NOT EDIT. From 77b95413b59130cdba014c10a86a16a37afdeb25 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 2 Aug 2022 15:10:57 +0200 Subject: [PATCH 65/89] wee adjustments --- internal/pkg/dl/migration.go | 114 +++++++++++++++++------------------ 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 0f7e41a1f..1b741377e 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -53,32 +53,27 @@ func Migrate(ctx context.Context, bulker bulk.Bulk) error { return nil } -func migrateTov7_15(ctx context.Context, bulker bulk.Bulk) error { - _, err := migrate(ctx, bulker, migrateAgentMetadata) - if err != nil { - return fmt.Errorf("v7.15.0 data migration failed: %w", err) - } - - return nil -} - -func migrateToV8_4(ctx context.Context, bulker bulk.Bulk) error { - migrated, err := migrate(ctx, bulker, migrateAgentOutputs) - if err != nil { - return fmt.Errorf("v8.4.0 data migration failed: %w", err) - } +func migrate(ctx context.Context, bulker bulk.Bulk, fn migrationBodyFn) (int, error) { + var updatedDocs int + for { + name, index, body, err := fn() + if err != nil { + return updatedDocs, fmt.Errorf(": %w", err) + } - // The migration was necessary and indeed run, thus we need to regenerate - // the API keys for all agents. In order to do so, we increase the policy - // coordinator index to force a policy update. - if migrated > 0 { - _, err := migrate(ctx, bulker, migratePolicyCoordinatorIdx) + resp, err := applyMigration(ctx, name, index, bulker, body) if err != nil { - return fmt.Errorf("v8.4.0 data migration failed: %w", err) + return updatedDocs, fmt.Errorf("failed to apply migration %q: %w", + name, err) + } + updatedDocs += resp.Updated + if resp.VersionConflicts == 0 { + break } - } - return nil + time.Sleep(time.Second) + } + return updatedDocs, nil } func applyMigration(ctx context.Context, name string, index string, bulker bulk.Bulk, body []byte) (migrationResponse, error) { @@ -139,27 +134,14 @@ func applyMigration(ctx context.Context, name string, index string, bulker bulk. return resp, err } -func migrate(ctx context.Context, bulker bulk.Bulk, fn migrationBodyFn) (int, error) { - var updatedDocs int - for { - name, index, body, err := fn() - if err != nil { - return updatedDocs, fmt.Errorf(": %w", err) - } - - resp, err := applyMigration(ctx, name, index, bulker, body) - if err != nil { - return updatedDocs, fmt.Errorf("failed to apply migration %q: %w", - name, err) - } - updatedDocs += resp.Updated - if resp.VersionConflicts == 0 { - break - } - - time.Sleep(time.Second) +// ============================== V7.15 migration ============================== +func migrateTov7_15(ctx context.Context, bulker bulk.Bulk) error { + _, err := migrate(ctx, bulker, migrateAgentMetadata) + if err != nil { + return fmt.Errorf("v7.15.0 data migration failed: %w", err) } - return updatedDocs, nil + + return nil } // FleetServer 7.15 added a new *AgentMetadata field to the Agent record. @@ -175,13 +157,13 @@ func migrate(ctx context.Context, bulker bulk.Bulk, fn migrationBodyFn) (int, er // As the update only occurs once, the 99.9% case is a noop. func migrateAgentMetadata() (string, string, []byte, error) { const migrationName = "AgentMetadata" - root := dsl.NewRoot() - root.Query().Bool().MustNot().Exists("agent.id") + query := dsl.NewRoot().Query().Bool().MustNot() + query.Exists("agent.id") painless := "ctx._source.agent = [:]; ctx._source.agent.id = ctx._id;" - root.Param("script", painless) + query.Param("script", painless) - body, err := root.MarshalJSON() + body, err := query.MarshalJSON() if err != nil { return migrationName, FleetAgents, nil, fmt.Errorf("could not marshal ES query: %w", err) } @@ -189,6 +171,28 @@ func migrateAgentMetadata() (string, string, []byte, error) { return migrationName, FleetAgents, body, nil } +// ============================== V8.4.0 migration ============================= +// https://github.com/elastic/fleet-server/issues/1672 + +func migrateToV8_4(ctx context.Context, bulker bulk.Bulk) error { + migrated, err := migrate(ctx, bulker, migrateAgentOutputs) + if err != nil { + return fmt.Errorf("v8.4.0 data migration failed: %w", err) + } + + // The migration was necessary and indeed run, thus we need to regenerate + // the API keys for all agents. In order to do so, we increase the policy + // coordinator index to force a policy update. + if migrated > 0 { + _, err := migrate(ctx, bulker, migratePolicyCoordinatorIdx) + if err != nil { + return fmt.Errorf("v8.4.0 data migration failed: %w", err) + } + } + + return nil +} + // migrateAgentOutputs performs the necessary changes on the Agent documents // to introduce the `Outputs` field. // @@ -205,8 +209,8 @@ func migrateAgentMetadata() (string, string, []byte, error) { func migrateAgentOutputs() (string, string, []byte, error) { const migrationName = "AgentOutputs" - root := dsl.NewRoot() - root.Query().Bool().MustNot().Exists("elasticsearch_outputs") + query := dsl.NewRoot().Query().Bool().MustNot() + query.Exists("elasticsearch_outputs") painless := ` // set up the new filed @@ -228,9 +232,9 @@ ctx._source.default_api_key=""; ctx._source.default_api_key_id=""; ctx._source.policy_output_permissions_hash=""; ` - root.Param("script", painless) + query.Param("script", painless) - body, err := root.MarshalJSON() + body, err := query.MarshalJSON() if err != nil { return migrationName, FleetAgents, nil, fmt.Errorf("could not marshal ES query: %w", err) } @@ -245,14 +249,10 @@ ctx._source.policy_output_permissions_hash=""; func migratePolicyCoordinatorIdx() (string, string, []byte, error) { const migrationName = "PolicyCoordinatorIdx" - root := dsl.NewRoot() - root.Query().MatchAll() - - painless := ` - ctx._source.coordinator_idx++;` - root.Param("script", painless) + query := dsl.NewRoot().Query().MatchAll() + query.Param("script", `ctx._source.coordinator_idx++;`) - body, err := root.MarshalJSON() + body, err := query.MarshalJSON() if err != nil { return migrationName, FleetPolicies, nil, fmt.Errorf("could not marshal ES query: %w", err) } From 959431c167ec3980ab978eb859466bbe51ccb5e8 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 2 Aug 2022 15:33:37 +0200 Subject: [PATCH 66/89] fix flaky test --- internal/pkg/api/handleAck_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/pkg/api/handleAck_test.go b/internal/pkg/api/handleAck_test.go index ece0f58f7..c95217cdf 100644 --- a/internal/pkg/api/handleAck_test.go +++ b/internal/pkg/api/handleAck_test.go @@ -464,7 +464,11 @@ func TestInvalidateAPIKeys(t *testing.T) { bulker := ftesting.NewMockBulk() bulker.On("APIKeyInvalidate", - context.Background(), want). + context.Background(), mock.MatchedBy(func(ids []string) bool { + // If A is a subset of B and B is a subset of A => A = B + return assert.Subset(t, ids, want) && + assert.Subset(t, want, ids) + })). Return(nil) ack := &AckT{bulk: bulker} From f522d01e28a7d21204575fbeea5fd2049873a1d3 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 2 Aug 2022 16:52:53 +0200 Subject: [PATCH 67/89] try to fix tests --- internal/pkg/dl/migration.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 1b741377e..80c880e30 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -157,8 +157,8 @@ func migrateTov7_15(ctx context.Context, bulker bulk.Bulk) error { // As the update only occurs once, the 99.9% case is a noop. func migrateAgentMetadata() (string, string, []byte, error) { const migrationName = "AgentMetadata" - query := dsl.NewRoot().Query().Bool().MustNot() - query.Exists("agent.id") + query := dsl.NewRoot() + query.Query().Bool().MustNot().Exists("agent.id") painless := "ctx._source.agent = [:]; ctx._source.agent.id = ctx._id;" query.Param("script", painless) @@ -209,8 +209,8 @@ func migrateToV8_4(ctx context.Context, bulker bulk.Bulk) error { func migrateAgentOutputs() (string, string, []byte, error) { const migrationName = "AgentOutputs" - query := dsl.NewRoot().Query().Bool().MustNot() - query.Exists("elasticsearch_outputs") + query := dsl.NewRoot() + query.Query().Bool().MustNot().Exists("elasticsearch_outputs") painless := ` // set up the new filed @@ -249,8 +249,8 @@ ctx._source.policy_output_permissions_hash=""; func migratePolicyCoordinatorIdx() (string, string, []byte, error) { const migrationName = "PolicyCoordinatorIdx" - query := dsl.NewRoot().Query().MatchAll() - query.Param("script", `ctx._source.coordinator_idx++;`) + query := dsl.NewRoot() + query.Query().MatchAll().Param("script", `ctx._source.coordinator_idx++;`) body, err := query.MarshalJSON() if err != nil { From 9d8ea7d4f19a79df14c336dcf60dcb862f1056ef Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 2 Aug 2022 17:13:04 +0200 Subject: [PATCH 68/89] try to fix tests --- internal/pkg/dl/migration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 80c880e30..0ff6706f7 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -250,7 +250,7 @@ func migratePolicyCoordinatorIdx() (string, string, []byte, error) { const migrationName = "PolicyCoordinatorIdx" query := dsl.NewRoot() - query.Query().MatchAll().Param("script", `ctx._source.coordinator_idx++;`) + query.Query().Bool().MatchAll().Param("script", `ctx._source.coordinator_idx++;`) body, err := query.MarshalJSON() if err != nil { From 1eb7bd73d1b4c8e1838095119549d0961752ae79 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 2 Aug 2022 17:23:06 +0200 Subject: [PATCH 69/89] fixing linter issues --- internal/pkg/coordinator/monitor_integration_test.go | 6 +++--- internal/pkg/dl/migration_integration_test.go | 4 ++-- internal/pkg/policy/policy_output_test.go | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/pkg/coordinator/monitor_integration_test.go b/internal/pkg/coordinator/monitor_integration_test.go index a43e3c1f0..74f86286c 100644 --- a/internal/pkg/coordinator/monitor_integration_test.go +++ b/internal/pkg/coordinator/monitor_integration_test.go @@ -417,13 +417,13 @@ func ensurePolicy(ctx context.Context, t *testing.T, bulker bulk.Bulk, index str break } } - if found == nil { + if found == nil { //nolint:staticcheck // false positive t.Fatal("policy not found") } - if found.RevisionIdx != revisionIdx { + if found.RevisionIdx != revisionIdx { //nolint:staticcheck // found is never nil t.Fatal("revision_idx does not match") } - if found.CoordinatorIdx != coordinatorIdx { + if found.CoordinatorIdx != coordinatorIdx { //nolint:staticcheck // found is never nil t.Fatal("coordinator_idx does not match") } } diff --git a/internal/pkg/dl/migration_integration_test.go b/internal/pkg/dl/migration_integration_test.go index 105ab5055..8ea5c8fd7 100644 --- a/internal/pkg/dl/migration_integration_test.go +++ b/internal/pkg/dl/migration_integration_test.go @@ -135,8 +135,8 @@ func TestPolicyCoordinatorIdx(t *testing.T) { func TestMigrateOutputs(t *testing.T) { index, bulker := ftesting.SetupCleanIndex(context.Background(), t, FleetAgents) apiKey := bulk.APIKey{ - ID: fmt.Sprint("testAgent_"), - Key: fmt.Sprint("testAgent_key_"), + ID: "testAgent_", + Key: "testAgent_key_", } agentIDs := createSomeAgents(t, 10, apiKey, index, bulker) diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go index d3e7e18c2..d0d6fc527 100644 --- a/internal/pkg/policy/policy_output_test.go +++ b/internal/pkg/policy/policy_output_test.go @@ -158,7 +158,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { Return(nil).Once() bulker.On("APIKeyCreate", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(&wantAPIKey, nil).Once() //nolint:govet // test case + Return(&wantAPIKey, nil).Once() output := Output{ Type: OutputTypeElasticsearch, @@ -220,7 +220,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { apiKey := bulk.APIKey{ID: "abc", Key: "new-key"} bulker.On("APIKeyCreate", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(&apiKey, nil).Once() //nolint:govet // test case + Return(&apiKey, nil).Once() output := Output{ Type: OutputTypeElasticsearch, From 4250ad6b4b22f83517a6e73d47e840cac06642eb Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 2 Aug 2022 17:30:12 +0200 Subject: [PATCH 70/89] try to fix tests --- .../pkg/coordinator/monitor_integration_test.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/internal/pkg/coordinator/monitor_integration_test.go b/internal/pkg/coordinator/monitor_integration_test.go index 74f86286c..fe9f76021 100644 --- a/internal/pkg/coordinator/monitor_integration_test.go +++ b/internal/pkg/coordinator/monitor_integration_test.go @@ -174,13 +174,14 @@ func TestMonitorUnenroller(t *testing.T) { // add agent that should be unenrolled sixAgo := time.Now().UTC().Add(-6 * time.Minute) agentBody, err := json.Marshal(model.Agent{ - AccessAPIKeyID: accessKey.ID, - DefaultAPIKeyID: outputKey.ID, - Active: true, - EnrolledAt: sixAgo.Format(time.RFC3339), - LastCheckin: sixAgo.Format(time.RFC3339), - PolicyID: policy1Id, - UpdatedAt: sixAgo.Format(time.RFC3339), + AccessAPIKeyID: accessKey.ID, + Outputs: map[string]*model.PolicyOutput{ + "default": {APIKeyID: outputKey.ID}}, + Active: true, + EnrolledAt: sixAgo.Format(time.RFC3339), + LastCheckin: sixAgo.Format(time.RFC3339), + PolicyID: policy1Id, + UpdatedAt: sixAgo.Format(time.RFC3339), }) require.NoError(t, err) _, err = bulker.Create(ctx, agentsIndex, agentID, agentBody) From 8fd7b62add3278ac801916baefb1c354d7af7945 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 3 Aug 2022 09:21:05 +0200 Subject: [PATCH 71/89] fix and adjust tests --- internal/pkg/dl/migration.go | 3 ++- internal/pkg/dl/migration_integration_test.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 0ff6706f7..d74f8ab09 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -250,7 +250,8 @@ func migratePolicyCoordinatorIdx() (string, string, []byte, error) { const migrationName = "PolicyCoordinatorIdx" query := dsl.NewRoot() - query.Query().Bool().MatchAll().Param("script", `ctx._source.coordinator_idx++;`) + query.Query().MatchAll() + query.Param("script", `ctx._source.coordinator_idx++;`) body, err := query.MarshalJSON() if err != nil { diff --git a/internal/pkg/dl/migration_integration_test.go b/internal/pkg/dl/migration_integration_test.go index 8ea5c8fd7..9ec9b75d1 100644 --- a/internal/pkg/dl/migration_integration_test.go +++ b/internal/pkg/dl/migration_integration_test.go @@ -139,7 +139,7 @@ func TestMigrateOutputs(t *testing.T) { Key: "testAgent_key_", } - agentIDs := createSomeAgents(t, 10, apiKey, index, bulker) + agentIDs := createSomeAgents(t, 25, apiKey, index, bulker) migratedAgents, err := migrate(context.Background(), bulker, migrateAgentOutputs) require.NoError(t, err) From e5f4b74ed7fcc54d163bd0ddd0a6e41fceea14da Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 3 Aug 2022 10:51:57 +0200 Subject: [PATCH 72/89] linter and PR fixes --- internal/pkg/apikey/apikey.go | 13 +++++++++---- internal/pkg/dl/constants.go | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/internal/pkg/apikey/apikey.go b/internal/pkg/apikey/apikey.go index fccc26f3d..bf28a4a16 100644 --- a/internal/pkg/apikey/apikey.go +++ b/internal/pkg/apikey/apikey.go @@ -6,6 +6,7 @@ package apikey import ( + "bytes" "context" "encoding/base64" "encoding/json" @@ -66,11 +67,16 @@ func Read(ctx context.Context, client *elasticsearch.Client, id string) (*APIKey APIKeys []APIKeyResponse `json:"api_keys"` } + var buff bytes.Buffer + if _, err := buff.ReadFrom(res.Body); err != nil { + return nil, fmt.Errorf("could not read from response body: %w", err) + } + defer res.Body.Close() + var resp GetAPIKeyResponse - d := json.NewDecoder(res.Body) - if err = d.Decode(&resp); err != nil { + if err = json.Unmarshal(buff.Bytes(), &resp); err != nil { return nil, fmt.Errorf( - "could not decode elasticsearch GetAPIKeyResponse: %w", err) + "could not Unmarshal elasticsearch GetAPIKeyResponse: %w", err) } if len(resp.APIKeys) == 0 { @@ -78,7 +84,6 @@ func Read(ctx context.Context, client *elasticsearch.Client, id string) (*APIKey } first := resp.APIKeys[0] - return &APIKeyMetadata{ ID: first.ID, Metadata: first.Metadata, diff --git a/internal/pkg/dl/constants.go b/internal/pkg/dl/constants.go index 494981301..24f643e05 100644 --- a/internal/pkg/dl/constants.go +++ b/internal/pkg/dl/constants.go @@ -37,9 +37,9 @@ const ( FieldPolicyCoordinatorIdx = "policy_coordinator_idx" FieldPolicyID = "policy_id" FieldPolicyOutputAPIKey = "api_key" - FieldPolicyOutputAPIKeyID = "api_key_id" //nolint:gosec // field name + FieldPolicyOutputAPIKeyID = "api_key_id" FieldPolicyOutputPermissionsHash = "policy_permissions_hash" - FieldPolicyOutputToRetireAPIKeys = "to_retire_api_keys" //nolint:gosec // field name + FieldPolicyOutputToRetireAPIKeys = "to_retire_api_keys" FieldPolicyRevisionIdx = "policy_revision_idx" FieldRevisionIdx = "revision_idx" FieldUnenrolledReason = "unenrolled_reason" From 0fb8c560afee46792826835015cb35eff70071a2 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 3 Aug 2022 10:57:03 +0200 Subject: [PATCH 73/89] change field name --- internal/pkg/dl/constants.go | 32 +++++++++---------- internal/pkg/dl/migration.go | 2 +- internal/pkg/model/schema.go | 8 ++--- internal/pkg/policy/policy_output.go | 6 ++-- .../policy/policy_output_integration_test.go | 6 ++-- model/schema.json | 8 ++--- 6 files changed, 31 insertions(+), 31 deletions(-) diff --git a/internal/pkg/dl/constants.go b/internal/pkg/dl/constants.go index 24f643e05..ff5242a8f 100644 --- a/internal/pkg/dl/constants.go +++ b/internal/pkg/dl/constants.go @@ -27,22 +27,22 @@ const ( FieldMaxSeqNo = "max_seq_no" FieldActionSeqNo = "action_seq_no" - FieldActionID = "action_id" - FieldAgent = "agent" - FieldAgentVersion = "version" - FieldCoordinatorIdx = "coordinator_idx" - FieldLastCheckin = "last_checkin" - FieldLastCheckinStatus = "last_checkin_status" - FieldLocalMetadata = "local_metadata" - FieldPolicyCoordinatorIdx = "policy_coordinator_idx" - FieldPolicyID = "policy_id" - FieldPolicyOutputAPIKey = "api_key" - FieldPolicyOutputAPIKeyID = "api_key_id" - FieldPolicyOutputPermissionsHash = "policy_permissions_hash" - FieldPolicyOutputToRetireAPIKeys = "to_retire_api_keys" - FieldPolicyRevisionIdx = "policy_revision_idx" - FieldRevisionIdx = "revision_idx" - FieldUnenrolledReason = "unenrolled_reason" + FieldActionID = "action_id" + FieldAgent = "agent" + FieldAgentVersion = "version" + FieldCoordinatorIdx = "coordinator_idx" + FieldLastCheckin = "last_checkin" + FieldLastCheckinStatus = "last_checkin_status" + FieldLocalMetadata = "local_metadata" + FieldPolicyCoordinatorIdx = "policy_coordinator_idx" + FieldPolicyID = "policy_id" + FieldPolicyOutputAPIKey = "api_key" + FieldPolicyOutputAPIKeyID = "api_key_id" + FieldPolicyOutputPermissionsHash = "policy_permissions_hash" + FieldPolicyOutputToRetireAPIKeyIDs = "to_retire_api_key_ids" + FieldPolicyRevisionIdx = "policy_revision_idx" + FieldRevisionIdx = "revision_idx" + FieldUnenrolledReason = "unenrolled_reason" FieldActive = "active" FieldUpdatedAt = "updated_at" diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index d74f8ab09..98aaa3186 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -221,7 +221,7 @@ if (ctx._source['outputs']['default']==null) // copy old values to new 'outputs' field ctx._source['outputs']['default'].type="elasticsearch"; -ctx._source['outputs']['default'].to_retire_api_keys=ctx._source.default_api_key_history; +ctx._source['outputs']['default'].to_retire_api_key_ids=ctx._source.default_api_key_history; ctx._source['outputs']['default'].api_key=ctx._source.default_api_key; ctx._source['outputs']['default'].api_key_id=ctx._source.default_api_key_id; ctx._source['outputs']['default'].policy_permissions_hash=ctx._source.policy_output_permissions_hash; diff --git a/internal/pkg/model/schema.go b/internal/pkg/model/schema.go index 5d71b850e..270d700f6 100644 --- a/internal/pkg/model/schema.go +++ b/internal/pkg/model/schema.go @@ -128,7 +128,7 @@ type Agent struct { DefaultAPIKey string `json:"default_api_key,omitempty"` // Deprecated. Use Outputs instead. Default API Key History - DefaultAPIKeyHistory []ToRetireAPIKeysItems `json:"default_api_key_history,omitempty"` + DefaultAPIKeyHistory []ToRetireAPIKeyIdsItems `json:"default_api_key_history,omitempty"` // Deprecated. Use Outputs instead. ID of the API key the Elastic Agent uses to authenticate with elasticsearch DefaultAPIKeyID string `json:"default_api_key_id,omitempty"` @@ -340,7 +340,7 @@ type PolicyOutput struct { PolicyPermissionsHash string `json:"policy_permissions_hash"` // API keys to be invalidated on next agent ack - ToRetireAPIKeys []ToRetireAPIKeysItems `json:"to_retire_api_keys,omitempty"` + ToRetireAPIKeyIds []ToRetireAPIKeyIdsItems `json:"to_retire_api_key_ids,omitempty"` // Type is the output type. Currently only Elasticsearch is supported. Type string `json:"type"` @@ -367,8 +367,8 @@ type ServerMetadata struct { Version string `json:"version"` } -// ToRetireAPIKeysItems the Output API Keys that were replaced and should be retired -type ToRetireAPIKeysItems struct { +// ToRetireAPIKeyIdsItems the Output API Keys that were replaced and should be retired +type ToRetireAPIKeyIdsItems struct { // API Key identifier ID string `json:"id,omitempty"` diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index dae75331d..f4b5f22e3 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -139,7 +139,7 @@ func (p *Output) prepareElasticsearch( dl.FieldPolicyOutputPermissionsHash: p.Role.Sha2, } if output.APIKeyID != "" { - fields[dl.FieldPolicyOutputToRetireAPIKeys] = model.ToRetireAPIKeysItems{ + fields[dl.FieldPolicyOutputToRetireAPIKeyIDs] = model.ToRetireAPIKeysItems{ ID: output.APIKeyID, RetiredAt: time.Now().UTC().Format(time.RFC3339), } @@ -185,8 +185,8 @@ if (ctx._source['outputs']['%s']==null) `, outputName, outputName)) for field := range fields { - if field == dl.FieldPolicyOutputToRetireAPIKeys { - // dl.FieldPolicyOutputToRetireAPIKeys is a special case. + if field == dl.FieldPolicyOutputToRetireAPIKeyIDs { + // dl.FieldPolicyOutputToRetireAPIKeyIDs is a special case. // It's an array that gets deleted when the keys are invalidated. // Thus, append the old API key ID, create the field if necessary. source.WriteString(fmt.Sprintf(` diff --git a/internal/pkg/policy/policy_output_integration_test.go b/internal/pkg/policy/policy_output_integration_test.go index 03d0cb840..c842f6bb7 100644 --- a/internal/pkg/policy/policy_output_integration_test.go +++ b/internal/pkg/policy/policy_output_integration_test.go @@ -29,10 +29,10 @@ func TestRenderUpdatePainlessScript(t *testing.T) { existingToRetireAPIKeys []model.ToRetireAPIKeysItems }{ { - name: "to_retire_api_keys is empty", + name: "to_retire_api_key_ids is empty", }, { - name: "to_retire_api_keys is not empty", + name: "to_retire_api_key_ids is not empty", existingToRetireAPIKeys: []model.ToRetireAPIKeysItems{{ ID: "pre_existing_ID", RetiredAt: "pre_existing__RetiredAt"}}, }, @@ -101,7 +101,7 @@ func TestRenderUpdatePainlessScript(t *testing.T) { dl.FieldPolicyOutputAPIKey: outputAPIKey.Agent(), dl.FieldPolicyOutputAPIKeyID: outputAPIKey.ID, dl.FieldPolicyOutputPermissionsHash: outputPermissionSha, - dl.FieldPolicyOutputToRetireAPIKeys: model.ToRetireAPIKeysItems{ + dl.FieldPolicyOutputToRetireAPIKeyIDs: model.ToRetireAPIKeysItems{ ID: previousAPIKey.ID, RetiredAt: nowStr}, } diff --git a/model/schema.json b/model/schema.json index 0cac892bd..9cdd46bed 100644 --- a/model/schema.json +++ b/model/schema.json @@ -350,7 +350,7 @@ ] }, - "to_retire_api_keys": { + "to_retire_api_key_ids": { "type": "array", "items": { "description": "the Output API Keys that were replaced and should be retired", @@ -377,9 +377,9 @@ "description": "API key the Elastic Agent uses to authenticate with elasticsearch", "type": "string" }, - "to_retire_api_keys": { + "to_retire_api_key_ids": { "description": "API keys to be invalidated on next agent ack", - "$ref": "#/definitions/to_retire_api_keys" + "$ref": "#/definitions/to_retire_api_key_ids" }, "api_key_id": { "description": "ID of the API key the Elastic Agent uses to authenticate with elasticsearch", @@ -522,7 +522,7 @@ }, "default_api_key_history": { "description": "Deprecated. Use Outputs instead. Default API Key History", - "$ref": "#/definitions/to_retire_api_keys" + "$ref": "#/definitions/to_retire_api_key_ids" }, "outputs": { "description": "Outputs is the policy output data, mapping the output name to its data", From 7a79d4c7875974cc2c3203ae1d10e0c6ed596532 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 3 Aug 2022 12:04:28 +0200 Subject: [PATCH 74/89] fix after renaming --- internal/pkg/api/handleAck.go | 4 ++-- internal/pkg/api/handleAck_test.go | 12 ++++++------ internal/pkg/dl/agent_integration_test.go | 2 +- internal/pkg/dl/migration_integration_test.go | 2 +- internal/pkg/model/ext.go | 2 +- internal/pkg/model/ext_test.go | 4 ++-- internal/pkg/policy/policy_output.go | 2 +- .../pkg/policy/policy_output_integration_test.go | 16 ++++++++-------- internal/pkg/policy/policy_output_test.go | 10 +++++----- 9 files changed, 27 insertions(+), 27 deletions(-) diff --git a/internal/pkg/api/handleAck.go b/internal/pkg/api/handleAck.go index be1374d2e..d9c599479 100644 --- a/internal/pkg/api/handleAck.go +++ b/internal/pkg/api/handleAck.go @@ -378,10 +378,10 @@ func (ack *AckT) handlePolicyChange(ctx context.Context, zlog zerolog.Logger, ag } func (ack *AckT) invalidateAPIKeys(ctx context.Context, agent *model.Agent) { - var toRetire []model.ToRetireAPIKeysItems + var toRetire []model.ToRetireAPIKeyIdsItems for _, out := range agent.Outputs { - toRetire = append(toRetire, out.ToRetireAPIKeys...) + toRetire = append(toRetire, out.ToRetireAPIKeyIds...) } size := len(toRetire) diff --git a/internal/pkg/api/handleAck_test.go b/internal/pkg/api/handleAck_test.go index c95217cdf..3613bd5f2 100644 --- a/internal/pkg/api/handleAck_test.go +++ b/internal/pkg/api/handleAck_test.go @@ -442,23 +442,23 @@ func TestHandleAckEvents(t *testing.T) { } func TestInvalidateAPIKeys(t *testing.T) { - toRetire1 := []model.ToRetireAPIKeysItems{{ + toRetire1 := []model.ToRetireAPIKeyIdsItems{{ ID: "toRetire1", }} - toRetire2 := []model.ToRetireAPIKeysItems{{ + toRetire2 := []model.ToRetireAPIKeyIdsItems{{ ID: "toRetire2_0", }, { ID: "toRetire2_1", }} - var toRetire3 []model.ToRetireAPIKeysItems + var toRetire3 []model.ToRetireAPIKeyIdsItems want := []string{"toRetire1", "toRetire2_0", "toRetire2_1"} agent := model.Agent{ Outputs: map[string]*model.PolicyOutput{ - "1": {ToRetireAPIKeys: toRetire1}, - "2": {ToRetireAPIKeys: toRetire2}, - "3": {ToRetireAPIKeys: toRetire3}, + "1": {ToRetireAPIKeyIds: toRetire1}, + "2": {ToRetireAPIKeyIds: toRetire2}, + "3": {ToRetireAPIKeyIds: toRetire3}, }, } diff --git a/internal/pkg/dl/agent_integration_test.go b/internal/pkg/dl/agent_integration_test.go index 6e883676c..2f8ad9ec1 100644 --- a/internal/pkg/dl/agent_integration_test.go +++ b/internal/pkg/dl/agent_integration_test.go @@ -122,7 +122,7 @@ func TestFindAgent_NewModel(t *testing.T) { "default": { Type: "elasticsearch", APIKey: "TestFindNewModelAgent_APIKey", - ToRetireAPIKeys: []model.ToRetireAPIKeysItems{ + ToRetireAPIKeyIds: []model.ToRetireAPIKeyIdsItems{ { ID: "TestFindNewModelAgent_APIKeyID_invalidated", RetiredAt: "TestFindNewModelAgent_APIKeyID_invalidated_at"}, diff --git a/internal/pkg/dl/migration_integration_test.go b/internal/pkg/dl/migration_integration_test.go index 9ec9b75d1..e70de61a3 100644 --- a/internal/pkg/dl/migration_integration_test.go +++ b/internal/pkg/dl/migration_integration_test.go @@ -49,7 +49,7 @@ func createSomeAgents(t *testing.T, n int, apiKey bulk.APIKey, index string, bul DefaultAPIKeyID: outputAPIKey.ID, DefaultAPIKey: outputAPIKey.Agent(), PolicyOutputPermissionsHash: fmt.Sprint("a_output_permission_SHA_", i), - DefaultAPIKeyHistory: []model.ToRetireAPIKeysItems{ + DefaultAPIKeyHistory: []model.ToRetireAPIKeyIdsItems{ { ID: "old_" + outputAPIKey.ID, RetiredAt: now.Add(-5 * time.Minute).Format(time.RFC3339), diff --git a/internal/pkg/model/ext.go b/internal/pkg/model/ext.go index 025415cea..4a11bbe08 100644 --- a/internal/pkg/model/ext.go +++ b/internal/pkg/model/ext.go @@ -52,7 +52,7 @@ func (a *Agent) APIKeyIDs() []string { for _, output := range a.Outputs { keys = append(keys, output.APIKeyID) - for _, key := range output.ToRetireAPIKeys { + for _, key := range output.ToRetireAPIKeyIds { keys = append(keys, key.ID) } } diff --git a/internal/pkg/model/ext_test.go b/internal/pkg/model/ext_test.go index 5f1667b72..527570270 100644 --- a/internal/pkg/model/ext_test.go +++ b/internal/pkg/model/ext_test.go @@ -108,12 +108,12 @@ func TestAgentAPIKeyIDs(t *testing.T) { Outputs: map[string]*PolicyOutput{ "p1": { APIKeyID: "p1_api_key_id", - ToRetireAPIKeys: []ToRetireAPIKeysItems{{ + ToRetireAPIKeyIds: []ToRetireAPIKeyIdsItems{{ ID: "p1_to_retire_key", }}}, "p2": { APIKeyID: "p2_api_key_id", - ToRetireAPIKeys: []ToRetireAPIKeysItems{{ + ToRetireAPIKeyIds: []ToRetireAPIKeyIdsItems{{ ID: "p2_to_retire_key", }}}, }, diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index f4b5f22e3..f0faceca1 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -139,7 +139,7 @@ func (p *Output) prepareElasticsearch( dl.FieldPolicyOutputPermissionsHash: p.Role.Sha2, } if output.APIKeyID != "" { - fields[dl.FieldPolicyOutputToRetireAPIKeyIDs] = model.ToRetireAPIKeysItems{ + fields[dl.FieldPolicyOutputToRetireAPIKeyIDs] = model.ToRetireAPIKeyIdsItems{ ID: output.APIKeyID, RetiredAt: time.Now().UTC().Format(time.RFC3339), } diff --git a/internal/pkg/policy/policy_output_integration_test.go b/internal/pkg/policy/policy_output_integration_test.go index c842f6bb7..1d301abca 100644 --- a/internal/pkg/policy/policy_output_integration_test.go +++ b/internal/pkg/policy/policy_output_integration_test.go @@ -26,14 +26,14 @@ func TestRenderUpdatePainlessScript(t *testing.T) { tts := []struct { name string - existingToRetireAPIKeys []model.ToRetireAPIKeysItems + existingToRetireAPIKeyIds []model.ToRetireAPIKeyIdsItems }{ { name: "to_retire_api_key_ids is empty", }, { name: "to_retire_api_key_ids is not empty", - existingToRetireAPIKeys: []model.ToRetireAPIKeysItems{{ + existingToRetireAPIKeyIds: []model.ToRetireAPIKeyIdsItems{{ ID: "pre_existing_ID", RetiredAt: "pre_existing__RetiredAt"}}, }, } @@ -63,8 +63,8 @@ func TestRenderUpdatePainlessScript(t *testing.T) { APIKeyID: outputAPIKey.ID, PolicyPermissionsHash: outputPermissionSha, Type: OutputTypeElasticsearch, - ToRetireAPIKeys: append(tt.existingToRetireAPIKeys, - model.ToRetireAPIKeysItems{ + ToRetireAPIKeyIds: append(tt.existingToRetireAPIKeyIds, + model.ToRetireAPIKeyIdsItems{ ID: previousAPIKey.ID, RetiredAt: nowStr}), }, } @@ -85,9 +85,9 @@ func TestRenderUpdatePainlessScript(t *testing.T) { }, }, } - if tt.existingToRetireAPIKeys != nil { - agentModel.Outputs[outputName].ToRetireAPIKeys = - tt.existingToRetireAPIKeys + if tt.existingToRetireAPIKeyIds != nil { + agentModel.Outputs[outputName].ToRetireAPIKeyIds = + tt.existingToRetireAPIKeyIds } body, err := json.Marshal(agentModel) @@ -101,7 +101,7 @@ func TestRenderUpdatePainlessScript(t *testing.T) { dl.FieldPolicyOutputAPIKey: outputAPIKey.Agent(), dl.FieldPolicyOutputAPIKeyID: outputAPIKey.ID, dl.FieldPolicyOutputPermissionsHash: outputPermissionSha, - dl.FieldPolicyOutputToRetireAPIKeyIDs: model.ToRetireAPIKeysItems{ + dl.FieldPolicyOutputToRetireAPIKeyIDs: model.ToRetireAPIKeyIdsItems{ ID: previousAPIKey.ID, RetiredAt: nowStr}, } diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go index d0d6fc527..beb49a917 100644 --- a/internal/pkg/policy/policy_output_test.go +++ b/internal/pkg/policy/policy_output_test.go @@ -109,7 +109,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { output.Name: { ESDocument: model.ESDocument{}, APIKey: apiKey.Agent(), - ToRetireAPIKeys: nil, + ToRetireAPIKeyIds: nil, APIKeyID: apiKey.ID, PolicyPermissionsHash: hashPerm, Type: OutputTypeElasticsearch, @@ -130,7 +130,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { assert.Equal(t, apiKey.ID, gotOutput.APIKeyID) assert.Equal(t, output.Role.Sha2, gotOutput.PolicyPermissionsHash) assert.Equal(t, output.Type, gotOutput.Type) - assert.Empty(t, gotOutput.ToRetireAPIKeys) + assert.Empty(t, gotOutput.ToRetireAPIKeyIds) // Old model must always remain empty assert.Empty(t, testAgent.DefaultAPIKey) @@ -178,7 +178,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { output.Name: { ESDocument: model.ESDocument{}, APIKey: oldAPIKey.Agent(), - ToRetireAPIKeys: nil, + ToRetireAPIKeyIds: nil, APIKeyID: oldAPIKey.ID, PolicyPermissionsHash: hashPerm, Type: OutputTypeElasticsearch, @@ -200,7 +200,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { assert.Equal(t, output.Role.Sha2, gotOutput.PolicyPermissionsHash) assert.Equal(t, output.Type, gotOutput.Type) - // assert.Contains(t, gotOutput.ToRetireAPIKeys, oldAPIKey.ID) // TODO: assert on bulker.Update + // assert.Contains(t, gotOutput.ToRetireAPIKeyIds, oldAPIKey.ID) // TODO: assert on bulker.Update // Old model must always remain empty assert.Empty(t, testAgent.DefaultAPIKey) @@ -250,7 +250,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { assert.Equal(t, apiKey.ID, gotOutput.APIKeyID) assert.Equal(t, output.Role.Sha2, gotOutput.PolicyPermissionsHash) assert.Equal(t, output.Type, gotOutput.Type) - assert.Empty(t, gotOutput.ToRetireAPIKeys) + assert.Empty(t, gotOutput.ToRetireAPIKeyIds) // Old model must always remain empty assert.Empty(t, testAgent.DefaultAPIKey) From 8ea0ede2da56e06813e302754b623c3a5be795fd Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 3 Aug 2022 12:15:47 +0200 Subject: [PATCH 75/89] fix linting issue --- internal/pkg/dl/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/dl/constants.go b/internal/pkg/dl/constants.go index ff5242a8f..037f0f05b 100644 --- a/internal/pkg/dl/constants.go +++ b/internal/pkg/dl/constants.go @@ -39,7 +39,7 @@ const ( FieldPolicyOutputAPIKey = "api_key" FieldPolicyOutputAPIKeyID = "api_key_id" FieldPolicyOutputPermissionsHash = "policy_permissions_hash" - FieldPolicyOutputToRetireAPIKeyIDs = "to_retire_api_key_ids" + FieldPolicyOutputToRetireAPIKeyIDs = "to_retire_api_key_ids" //nolint:gosec // false positive FieldPolicyRevisionIdx = "policy_revision_idx" FieldRevisionIdx = "revision_idx" FieldUnenrolledReason = "unenrolled_reason" From 31d0537853afb895ad7ffd8b73dc7f74ddfc0b43 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 3 Aug 2022 14:52:44 +0200 Subject: [PATCH 76/89] . --- 0.notes.md | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/0.notes.md b/0.notes.md index 6d0936313..a100232f3 100644 --- a/0.notes.md +++ b/0.notes.md @@ -1,17 +1,9 @@ -1. Add fields to the .agent fields, don't change the existing ones -2. on upgrade bump the coordinator index -3. on upgrade set the default api to empty -> force all API keys to be regenerated -4. on agent checkin, after upgrade, try to search api key by metadata.agentId and invalidate the "old" ones -5. ensure we delete the old API keys -6. fix painless script -7. add output name or something to api key metadata +How to identify and reproduce +---------------------------------------------------------------------------------------------------- +TODO(@AndersonQ): delete it before merge - - -Thu 14 Jul 16:09:41 CEST 2022 - -so far I could drill the problem down to how the output permissions change. +so far (Thu 14 Jul 16:09:41 CEST 2022) I could drill the problem down to how the output permissions change. It seems that depending on the added/removed/changed integration it might or might not affect the default output (the output for the agent monitoring logs and metrics) From 6e5c500c38e70fb6dabd0e38ad3bec9b9e5ab789 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 3 Aug 2022 14:54:17 +0200 Subject: [PATCH 77/89] . --- 0.notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/0.notes.md b/0.notes.md index a100232f3..12aa65b88 100644 --- a/0.notes.md +++ b/0.notes.md @@ -1,4 +1,4 @@ -How to identify and reproduce +How to identify and reproduce [#1672](https://github.com/elastic/fleet-server/issues/1672) ---------------------------------------------------------------------------------------------------- TODO(@AndersonQ): delete it before merge From 3124b029888a5f998c97cf50dbe1b438fa048981 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 4 Aug 2022 11:19:31 +0200 Subject: [PATCH 78/89] delete 0.notes.md, using a gist instead --- 0.notes.md | 1220 ---------------------------------------------------- 1 file changed, 1220 deletions(-) delete mode 100644 0.notes.md diff --git a/0.notes.md b/0.notes.md deleted file mode 100644 index 12aa65b88..000000000 --- a/0.notes.md +++ /dev/null @@ -1,1220 +0,0 @@ -How to identify and reproduce [#1672](https://github.com/elastic/fleet-server/issues/1672) ----------------------------------------------------------------------------------------------------- - -TODO(@AndersonQ): delete it before merge - -so far (Thu 14 Jul 16:09:41 CEST 2022) I could drill the problem down to how the output permissions change. -It seems that depending on the added/removed/changed integration it might or might -not affect the default output (the output for the agent monitoring logs and metrics) - - - if it affects both (I'm working only with 2 outputs): all good. 2 different apy keys are generated - - if it affects only one (here the not default one) output: the problem happens. -Only one new API key is generated, and both outputs use the same, which does not have enough permissions for both outputs - -One thing that seems to indicate the problem happened is the log `Invalidate old API keys` -where the `ids` field has got repeated key IDs, like this one: -```json -[{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"DhkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[18638],"message":["Invalidate old API keys"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:41.952Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"ids":["KRkB_YEBMKjCD54NTy4Q","KRkB_YEBMKjCD54NTy4Q"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808321952]}] -``` - - -#### Only one API key being generated -Endpoint security was removed - -- Elastic-Agent inspect -```text -outputs: - 6d0e50a0-0338-11ed-849a-2dafbb876867: - api_key: KRkB_YEBMKjCD54NTy4Q: - bulk_max_size: 250 - hosts: - - https://192.168.56.1:9200 - ssl: - ca_trusted_fingerprint: 7a145ed00941a323a0d18351e179319a107ee58e07cd7165c72d087b9d685c1f - type: elasticsearch - workers: 8 - default: - api_key: KRkB_YEBMKjCD54NTy4Q: - hosts: - - https://192.168.56.1:9200 - ssl: - ca_trusted_fingerprint: 7a145ed00941a323a0d18351e179319a107ee58e07cd7165c72d087b9d685c1f - type: elasticsearch -revision: 18 -``` -
- Fleet-server logs (click to expand) - -```json -[{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"zhkB_YEBMKjCD54NpDjo","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[55539],"message":["Invalidate old API keys"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:59:12.000Z"],"@timestamp":["2022-07-14T13:59:06.732Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"ids":["Cxn9_IEBMKjCD54N3RVD"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807146732]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"shkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[53496],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["preparing elasticsearch output"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.681Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130681]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"sxkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[54017],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["policy output permissions are the same"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.681Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130681]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"tBkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"host.hostname":["fleet-server-dev"],"type":["POLICY_CHANGE"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"timeout":[0],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"createdAt":["2022-07-14T13:58:48.878Z"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"inputType":[""],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"id":["policy:67397b70-0354-11ed-849a-2dafbb876867:18:1"],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[54546],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Action delivered to agent on checkin"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.681Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"ackToken":[""],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130681]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"rhkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"fleet.anderson.default.old.apikey":["Cxn9_IEBMKjCD54N3RVD:8TNfbXQqQf2GWErLbHgKOg"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"fleet.anderson.default.new.apikey":["KRkB_YEBMKjCD54NTy4Q:"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[50786],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.404Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130404]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"rxkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.anderson_old_apikey":["Cxn9_IEBMKjCD54N3RVD:8TNfbXQqQf2GWErLbHgKOg"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"fleet.anderson_new_apikey":["KRkB_YEBMKjCD54NTy4Q:"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[51510],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.404Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130404]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"sBkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[52218],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.404Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130404]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"sRkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"fleet.role.hash.sha256":["3da93ea478a1ae088c235461ae13adac7cbca1db0ff5319c8e3bf9421bae8a90"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"fleet.default.apikey.id":["KRkB_YEBMKjCD54NTy4Q"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[52783],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Updating agent record to pick up default output key."],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.404Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130404]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"rRkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[50221],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.403Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130403]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"qRkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[45913],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["preparing elasticsearch output"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.383Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130383]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"qhkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[46463],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["must generate api key as policy output permissions changed"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.383Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130383]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"rBkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YFW3YM2TY83P4YZ7S54PGZ"],"policyRevision":[18],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[49552],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["generating output API key b91f9075-5f1b-45f1-949c-d7b852e88b7a:6d0e50a0-0338-11ed-849a-2dafbb876867 for agent ID b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.383Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130383]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"qBkB_YEBMKjCD54Nbi41","_version":1,"_score":null,"fields":{"oldCoord":[1],"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"nQueued":[1],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"coord":[1],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"rev":[18],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["policy agent monitor"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[45534],"message":["New revision of policy received and added to the queue"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T13:58:58.000Z"],"@timestamp":["2022-07-14T13:58:50.129Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"oldRev":[17],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-39.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657807130129]}] -``` -
- -
- API Key doc on ES (click to expand) - -```json -{ - "_index": ".security-7", - "_id": "KRkB_YEBMKjCD54NTy4Q", - "_version": 1, - "_seq_no": 275, - "_primary_term": 1, - "found": true, - "_source": { - "doc_type": "api_key", - "creation_time": 1657807130384, - "expiration_time": null, - "api_key_invalidated": false, - "api_key_hash": "{PBKDF2}10000$cExNjhZ5MHlYi+zD6kMjQDkR7cDsCKlHXaLmnaQExkg=$gGv9yrq+svB2+HBnvYhD90L+X46e2gkpsTy2kQQMyiM=", - "role_descriptors": { - "4e80bb57-d538-48d0-9534-93f292e5fa22": { - "cluster": [], - "indices": [ - { - "names": [ - "logs-aws.vpcflow-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - } - ], - "applications": [], - "run_as": [], - "metadata": {}, - "type": "role" - }, - "_elastic_agent_checks": { - "cluster": [ - "monitor" - ], - "indices": [], - "applications": [], - "run_as": [], - "metadata": {}, - "type": "role" - }, - "a1157d27-35ff-4cb0-a4dc-28e21418ebb9": { - "cluster": [], - "indices": [ - { - "names": [ - "logs-generic-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - } - ], - "applications": [], - "run_as": [], - "metadata": {}, - "type": "role" - }, - "daab270e-6fe8-446e-8176-c877fe9e73da": { - "cluster": [], - "indices": [ - { - "names": [ - "logs-system.auth-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-system.syslog-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-system.application-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-system.security-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-system.system-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.cpu-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.diskio-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.filesystem-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.fsstat-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.load-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.memory-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.network-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.process-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.process.summary-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.socket_summary-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.uptime-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - } - ], - "applications": [], - "run_as": [], - "metadata": {}, - "type": "role" - } - }, - "limited_by_role_descriptors": { - "elastic/fleet-server": { - "cluster": [ - "monitor", - "manage_own_api_key" - ], - "indices": [ - { - "names": [ - "logs-*", - "metrics-*", - "traces-*", - "synthetics-*", - ".logs-endpoint.diagnostic.collection-*", - ".logs-endpoint.action.responses-*" - ], - "privileges": [ - "write", - "create_index", - "auto_configure" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "traces-apm.sampled-*" - ], - "privileges": [ - "read", - "monitor", - "maintenance" - ], - "allow_restricted_indices": false - }, - { - "names": [ - ".fleet-*" - ], - "privileges": [ - "read", - "write", - "monitor", - "create_index", - "auto_configure", - "maintenance" - ], - "allow_restricted_indices": true - } - ], - "applications": [ - { - "application": "kibana-*", - "privileges": [ - "reserved_fleet-setup" - ], - "resources": [ - "*" - ] - } - ], - "run_as": [], - "metadata": {}, - "type": "role" - } - }, - "name": "b91f9075-5f1b-45f1-949c-d7b852e88b7a:6d0e50a0-0338-11ed-849a-2dafbb876867", - "version": 8030199, - "metadata_flattened": { - "agent_id": "b91f9075-5f1b-45f1-949c-d7b852e88b7a", - "managed_by": "fleet-server", - "managed": true, - "type": "output" - }, - "creator": { - "principal": "elastic/fleet-server", - "full_name": "Service account - elastic/fleet-server", - "email": null, - "metadata": { - "_elastic_service_account": true - }, - "realm": "_service_account", - "realm_type": "_service_account" - } - } -} -``` -
- -#### Both API keys being generated -Endpoint security was added - -- Elastic-Agent inspect -```text -outputs: - 6d0e50a0-0338-11ed-849a-2dafbb876867: - api_key: VBkT_YEBMKjCD54NWmW4: - bulk_max_size: 250 - hosts: - - https://192.168.56.1:9200 - ssl: - ca_trusted_fingerprint: 7a145ed00941a323a0d18351e179319a107ee58e07cd7165c72d087b9d685c1f - type: elasticsearch - workers: 8 - default: - api_key: VhkT_YEBMKjCD54NW2XR: - hosts: - - https://192.168.56.1:9200 - ssl: - ca_trusted_fingerprint: 7a145ed00941a323a0d18351e179319a107ee58e07cd7165c72d087b9d685c1f - type: elasticsearch -revision: 19 - -``` - -
- Fleet-server logs (click to expand) - -```json -[{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"-RkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[797],"message":["bb876867\",\"rev\":19,\"coord\":1,\"oldRev\":18,\"oldCoord\":1,\"nQueued\":1,\"fleet.policy.id\":\"67397b70-0354-11ed-849a-2dafbb876867\",\"@timestamp\":\"2022-07-14T14:18:32.745Z\",\"message\":\"New revision of policy received and added to the queue\"}"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:50.393Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808330393]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"DxkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH6Z3QP79ZTPS3DM1MVWKJ"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[18841],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["ack policy"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:42.798Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"nEvents":[1],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808322798]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"DhkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[18638],"message":["Invalidate old API keys"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:41.952Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"ids":["KRkB_YEBMKjCD54NTy4Q","KRkB_YEBMKjCD54NTy4Q"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808321952]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"DRkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"agentId":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH6Z3QP79ZTPS3DM1MVWKJ"],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"actionSubType":["ACKNOWLEDGED"],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"timestamp":["2022-07-14T14:18:41.65523+00:00"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[18117],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["ack event"],"data_stream.type":["logs"],"n":[0],"host.architecture":["x86_64"],"actionType":["ACTION_RESULT"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:41.951Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"actionId":["policy:67397b70-0354-11ed-849a-2dafbb876867:19:1"],"agent.version":["8.3.1"],"host.os.family":["debian"],"nEvents":[1],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808321951]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"DBkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"host.hostname":["fleet-server-dev"],"type":["POLICY_CHANGE"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"timeout":[0],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"createdAt":["2022-07-14T14:18:31.609Z"],"agent.type":["filebeat"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"inputType":[""],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"id":["policy:67397b70-0354-11ed-849a-2dafbb876867:19:1"],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[17645],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Action delivered to agent on checkin"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.608Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"ackToken":[""],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313608]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"BxkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[14515],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"CBkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"fleet.anderson.default.old.apikey":["VBkT_YEBMKjCD54NWmW4:"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"fleet.anderson.default.new.apikey":["VhkT_YEBMKjCD54NW2XR:"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[15051],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"CRkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.anderson_old_apikey":["VBkT_YEBMKjCD54NWmW4:"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"fleet.anderson_new_apikey":["VhkT_YEBMKjCD54NW2XR:"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[15746],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"ChkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[16425],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"CxkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"fleet.role.hash.sha256":["ac0c515dcc1ef486f784feccdb11c88cd84cd1d2e869a66bc1dad6d29b837a0a"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"fleet.default.apikey.id":["VhkT_YEBMKjCD54NW2XR"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[16961],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Updating agent record to pick up default output key."],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.316Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313316]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"AxkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[9896],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["preparing elasticsearch output"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.296Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313296]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"BBkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[10417],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["must generate api key as policy output permissions changed"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.296Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313296]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"BhkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["default"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[13904],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["generating output API key b91f9075-5f1b-45f1-949c-d7b852e88b7a:default for agent ID b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.296Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313296]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"_hkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[6621],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"_xkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"fleet.anderson.default.old.apikey":["KRkB_YEBMKjCD54NTy4Q:"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"fleet.anderson.default.new.apikey":["VBkT_YEBMKjCD54NWmW4:"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[7186],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"ABkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.anderson_old_apikey":["KRkB_YEBMKjCD54NTy4Q:"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"fleet.anderson_new_apikey":["VBkT_YEBMKjCD54NWmW4:"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[7910],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["setting / swapping agent default API key"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"ARkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"ctx":["processPolicy"],"policyCoordinator":[1],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[8618],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["============================================="],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"AhkT_YEBMKjCD54NonGG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"fleet.role.hash.sha256":["a4af23b737b102e5e4555a6a062531fe08660d096e5c7aa9ed73d61fdf7ff5b6"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"fleet.default.apikey.id":["VBkT_YEBMKjCD54NWmW4"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[9183],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["Updating agent record to pick up default output key."],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.034Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313034]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"-hkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[1028],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["preparing elasticsearch output"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.015Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313015]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"-xkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[1578],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["must generate api key as policy output permissions changed"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.015Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313015]},{"_index":".ds-logs-elastic_agent.fleet_server-default-2022.07.14-000001","_id":"_RkT_YEBMKjCD54NonCG","_version":1,"_score":null,"fields":{"elastic_agent.version":["8.3.1"],"host.os.name.text":["Ubuntu"],"http.request.id":["01G7YH66NKF8BK9H80GK4R4EYR"],"policyRevision":[19],"host.hostname":["fleet-server-dev"],"host.mac":["02:6c:bf:6f:0c:40","08:00:27:d8:31:39"],"host.ip":["10.0.2.15","fe80::6c:bfff:fe6f:c40","192.168.56.43","fe80::a00:27ff:fed8:3139"],"agent.type":["filebeat"],"fleet.policy.id":["67397b70-0354-11ed-849a-2dafbb876867"],"host.os.version":["21.10 (Impish Indri)"],"host.os.kernel":["5.13.0-52-generic"],"host.os.name":["Ubuntu"],"log.level":["info"],"agent.name":["fleet-server-dev"],"host.name":["fleet-server-dev"],"elastic_agent.snapshot":[false],"event.agent_id_status":["verified"],"host.id":["c287e219ca4f4643aa206fc914242e26"],"fleet.policy.output.name":["6d0e50a0-0338-11ed-849a-2dafbb876867"],"fleet.agent.id":["b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"service.name":["fleet-server"],"host.os.type":["linux"],"elastic_agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"data_stream.namespace":["default"],"policyCoordinator":[1],"ctx":["processPolicy"],"host.os.codename":["impish"],"input.type":["filestream"],"log.offset":[5952],"fleet.access.apikey.id":["Dw8G_IEBMKjCD54N1Ziy"],"message":["generating output API key b91f9075-5f1b-45f1-949c-d7b852e88b7a:6d0e50a0-0338-11ed-849a-2dafbb876867 for agent ID b91f9075-5f1b-45f1-949c-d7b852e88b7a"],"data_stream.type":["logs"],"host.architecture":["x86_64"],"event.ingested":["2022-07-14T14:18:51.000Z"],"@timestamp":["2022-07-14T14:18:33.015Z"],"agent.id":["919e52ba-29d1-4783-b9f8-f9de8073cddc"],"host.os.platform":["ubuntu"],"ecs.version":["8.0.0"],"host.containerized":[false],"data_stream.dataset":["elastic_agent.fleet_server"],"log.file.path":["/opt/Elastic/Agent/data/elastic-agent-190a5b/logs/default/fleet-server-20220714-40.ndjson"],"agent.ephemeral_id":["409d0517-894b-4510-838c-874d17284518"],"agent.version":["8.3.1"],"host.os.family":["debian"],"event.dataset":["elastic_agent.fleet_server"]},"sort":[1657808313015]}] -``` -
- -
- API Key docs on ES (click to expand) - -- output 6d0e50a0-0338-11ed-849a-2dafbb876867, key id VBkT_YEBMKjCD54NWmW4 -```json -{ - "_index": ".security-7", - "_id": "VBkT_YEBMKjCD54NWmW4", - "_version": 1, - "_seq_no": 277, - "_primary_term": 1, - "found": true, - "_source": { - "doc_type": "api_key", - "creation_time": 1657808313016, - "expiration_time": null, - "api_key_invalidated": false, - "api_key_hash": "{PBKDF2}10000$jrCIMrbr1+zDVZw363AkxtnkkpavDr/SNgtr2wiU69Q=$Z5bwX7LA0fVmq99KwaMEDSqaYIU8lwhD1LTqct5etAw=", - "role_descriptors": { - "4e80bb57-d538-48d0-9534-93f292e5fa22": { - "cluster": [], - "indices": [ - { - "names": [ - "logs-aws.vpcflow-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - } - ], - "applications": [], - "run_as": [], - "metadata": {}, - "type": "role" - }, - "6d72b1a4-dc33-4d42-b92a-744bb1ef364e": { - "cluster": [], - "indices": [ - { - "names": [ - ".logs-endpoint.action.responses-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - ".logs-endpoint.actions-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-endpoint.alerts-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - ".logs-endpoint.diagnostic.collection-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-endpoint.events.file-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-endpoint.events.library-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-endpoint.metadata-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-endpoint.metrics-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-endpoint.events.network-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-endpoint.policy-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-endpoint.events.process-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-endpoint.events.registry-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-endpoint.events.security-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - } - ], - "applications": [], - "run_as": [], - "metadata": {}, - "type": "role" - }, - "_elastic_agent_checks": { - "cluster": [ - "monitor" - ], - "indices": [], - "applications": [], - "run_as": [], - "metadata": {}, - "type": "role" - }, - "a1157d27-35ff-4cb0-a4dc-28e21418ebb9": { - "cluster": [], - "indices": [ - { - "names": [ - "logs-generic-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - } - ], - "applications": [], - "run_as": [], - "metadata": {}, - "type": "role" - }, - "daab270e-6fe8-446e-8176-c877fe9e73da": { - "cluster": [], - "indices": [ - { - "names": [ - "logs-system.auth-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-system.syslog-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-system.application-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-system.security-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-system.system-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.cpu-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.diskio-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.filesystem-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.fsstat-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.load-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.memory-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.network-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.process-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.process.summary-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.socket_summary-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-system.uptime-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - } - ], - "applications": [], - "run_as": [], - "metadata": {}, - "type": "role" - } - }, - "limited_by_role_descriptors": { - "elastic/fleet-server": { - "cluster": [ - "monitor", - "manage_own_api_key" - ], - "indices": [ - { - "names": [ - "logs-*", - "metrics-*", - "traces-*", - "synthetics-*", - ".logs-endpoint.diagnostic.collection-*", - ".logs-endpoint.action.responses-*" - ], - "privileges": [ - "write", - "create_index", - "auto_configure" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "traces-apm.sampled-*" - ], - "privileges": [ - "read", - "monitor", - "maintenance" - ], - "allow_restricted_indices": false - }, - { - "names": [ - ".fleet-*" - ], - "privileges": [ - "read", - "write", - "monitor", - "create_index", - "auto_configure", - "maintenance" - ], - "allow_restricted_indices": true - } - ], - "applications": [ - { - "application": "kibana-*", - "privileges": [ - "reserved_fleet-setup" - ], - "resources": [ - "*" - ] - } - ], - "run_as": [], - "metadata": {}, - "type": "role" - } - }, - "name": "b91f9075-5f1b-45f1-949c-d7b852e88b7a:6d0e50a0-0338-11ed-849a-2dafbb876867", - "version": 8030199, - "metadata_flattened": { - "agent_id": "b91f9075-5f1b-45f1-949c-d7b852e88b7a", - "managed_by": "fleet-server", - "managed": true, - "type": "output" - }, - "creator": { - "principal": "elastic/fleet-server", - "full_name": "Service account - elastic/fleet-server", - "email": null, - "metadata": { - "_elastic_service_account": true - }, - "realm": "_service_account", - "realm_type": "_service_account" - } - } -} -``` -
- -
- output default, key id VhkT_YEBMKjCD54NW2XR (click to expand) - -```json -{ - "_index": ".security-7", - "_id": "VhkT_YEBMKjCD54NW2XR", - "_version": 1, - "_seq_no": 278, - "_primary_term": 1, - "found": true, - "_source": { - "doc_type": "api_key", - "creation_time": 1657808313298, - "expiration_time": null, - "api_key_invalidated": false, - "api_key_hash": "{PBKDF2}10000$+/HtACFwZUwk/31JJw7j2Lk+aQtvmlIsdvUzqHMF/Pw=$3CVORNqyrjhg0Xvfdd6k5MXOWmS1Cf/d9QvnOXiXTkM=", - "role_descriptors": { - "_elastic_agent_checks": { - "cluster": [ - "monitor" - ], - "indices": [], - "applications": [], - "run_as": [], - "metadata": {}, - "type": "role" - }, - "_elastic_agent_monitoring": { - "cluster": [], - "indices": [ - { - "names": [ - "logs-elastic_agent.apm_server-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-elastic_agent.apm_server-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-elastic_agent.auditbeat-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-elastic_agent.auditbeat-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-elastic_agent.cloudbeat-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-elastic_agent.cloudbeat-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-elastic_agent-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-elastic_agent.elastic_agent-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-elastic_agent.endpoint_security-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-elastic_agent.endpoint_security-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-elastic_agent.filebeat-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-elastic_agent.filebeat-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-elastic_agent.fleet_server-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-elastic_agent.fleet_server-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-elastic_agent.heartbeat-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-elastic_agent.heartbeat-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-elastic_agent.metricbeat-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-elastic_agent.metricbeat-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-elastic_agent.osquerybeat-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-elastic_agent.osquerybeat-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "logs-elastic_agent.packetbeat-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "metrics-elastic_agent.packetbeat-default" - ], - "privileges": [ - "auto_configure", - "create_doc" - ], - "allow_restricted_indices": false - } - ], - "applications": [], - "run_as": [], - "metadata": {}, - "type": "role" - } - }, - "limited_by_role_descriptors": { - "elastic/fleet-server": { - "cluster": [ - "monitor", - "manage_own_api_key" - ], - "indices": [ - { - "names": [ - "logs-*", - "metrics-*", - "traces-*", - "synthetics-*", - ".logs-endpoint.diagnostic.collection-*", - ".logs-endpoint.action.responses-*" - ], - "privileges": [ - "write", - "create_index", - "auto_configure" - ], - "allow_restricted_indices": false - }, - { - "names": [ - "traces-apm.sampled-*" - ], - "privileges": [ - "read", - "monitor", - "maintenance" - ], - "allow_restricted_indices": false - }, - { - "names": [ - ".fleet-*" - ], - "privileges": [ - "read", - "write", - "monitor", - "create_index", - "auto_configure", - "maintenance" - ], - "allow_restricted_indices": true - } - ], - "applications": [ - { - "application": "kibana-*", - "privileges": [ - "reserved_fleet-setup" - ], - "resources": [ - "*" - ] - } - ], - "run_as": [], - "metadata": {}, - "type": "role" - } - }, - "name": "b91f9075-5f1b-45f1-949c-d7b852e88b7a:default", - "version": 8030199, - "metadata_flattened": { - "agent_id": "b91f9075-5f1b-45f1-949c-d7b852e88b7a", - "managed_by": "fleet-server", - "managed": true, - "type": "output" - }, - "creator": { - "principal": "elastic/fleet-server", - "full_name": "Service account - elastic/fleet-server", - "email": null, - "metadata": { - "_elastic_service_account": true - }, - "realm": "_service_account", - "realm_type": "_service_account" - } - } -} -``` - -
\ No newline at end of file From b87920f486463af1292be906a97bde135a25fc61 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 10 Aug 2022 19:03:36 +0200 Subject: [PATCH 79/89] address PR comments --- internal/pkg/api/handleAck.go | 14 +++++--------- internal/pkg/apikey/apikey.go | 1 - internal/pkg/testing/setup.go | 2 +- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/internal/pkg/api/handleAck.go b/internal/pkg/api/handleAck.go index d9c599479..58b66fa83 100644 --- a/internal/pkg/api/handleAck.go +++ b/internal/pkg/api/handleAck.go @@ -378,18 +378,14 @@ func (ack *AckT) handlePolicyChange(ctx context.Context, zlog zerolog.Logger, ag } func (ack *AckT) invalidateAPIKeys(ctx context.Context, agent *model.Agent) { - var toRetire []model.ToRetireAPIKeyIdsItems - + var ids []string for _, out := range agent.Outputs { - toRetire = append(toRetire, out.ToRetireAPIKeyIds...) + for _, k := range out.ToRetireAPIKeyIds { + ids = append(ids, k.ID) + } } - size := len(toRetire) - if size > 0 { - ids := make([]string, size) - for i := 0; i < size; i++ { - ids[i] = toRetire[i].ID - } + if len(ids) > 0 { log.Info().Strs("fleet.policy.apiKeyIDsToRetire", ids).Msg("Invalidate old API keys") if err := ack.bulk.APIKeyInvalidate(ctx, ids...); err != nil { log.Info().Err(err).Strs("ids", ids).Msg("Failed to invalidate API keys") diff --git a/internal/pkg/apikey/apikey.go b/internal/pkg/apikey/apikey.go index bf28a4a16..4134f2b0d 100644 --- a/internal/pkg/apikey/apikey.go +++ b/internal/pkg/apikey/apikey.go @@ -71,7 +71,6 @@ func Read(ctx context.Context, client *elasticsearch.Client, id string) (*APIKey if _, err := buff.ReadFrom(res.Body); err != nil { return nil, fmt.Errorf("could not read from response body: %w", err) } - defer res.Body.Close() var resp GetAPIKeyResponse if err = json.Unmarshal(buff.Bytes(), &resp); err != nil { diff --git a/internal/pkg/testing/setup.go b/internal/pkg/testing/setup.go index 898afa36b..8f38ba7e6 100644 --- a/internal/pkg/testing/setup.go +++ b/internal/pkg/testing/setup.go @@ -106,7 +106,7 @@ func CleanIndex(ctx context.Context, t *testing.T, bulker bulk.Bulk, index strin query, err := q.Render(make(map[string]interface{})) if err != nil { - t.Fatalf("could not clena index: failed t render query template: %v", err) + t.Fatalf("could not clean index: failed to render query template: %v", err) } cli := bulker.Client() From 6f0c0db40d140892048b83ad6107940cd8fdcc84 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Fri, 12 Aug 2022 11:55:29 +0200 Subject: [PATCH 80/89] make migration to run --- cmd/fleet/main.go | 5 ++++- internal/pkg/dl/migration.go | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/fleet/main.go b/cmd/fleet/main.go index c7634d2e7..b87ffcb71 100644 --- a/cmd/fleet/main.go +++ b/cmd/fleet/main.go @@ -811,9 +811,12 @@ func (f *FleetServer) runSubsystems(ctx context.Context, cfg *config.Config, g * } // Run migrations - loggedRunFunc(ctx, "Migrations", func(ctx context.Context) error { + loggedMigration := loggedRunFunc(ctx, "Migrations", func(ctx context.Context) error { return dl.Migrate(ctx, bulker) }) + if err = loggedMigration(); err != nil { + return fmt.Errorf("failed to run subsystems: %w", err) + } // Run scheduler for periodic GC/cleanup gcCfg := cfg.Inputs[0].Server.GC diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 98aaa3186..1ab989c29 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -136,6 +136,7 @@ func applyMigration(ctx context.Context, name string, index string, bulker bulk. // ============================== V7.15 migration ============================== func migrateTov7_15(ctx context.Context, bulker bulk.Bulk) error { + log.Info().Msg("applying migration to v7.15") _, err := migrate(ctx, bulker, migrateAgentMetadata) if err != nil { return fmt.Errorf("v7.15.0 data migration failed: %w", err) @@ -175,6 +176,7 @@ func migrateAgentMetadata() (string, string, []byte, error) { // https://github.com/elastic/fleet-server/issues/1672 func migrateToV8_4(ctx context.Context, bulker bulk.Bulk) error { + log.Info().Msg("applying migration to v8.4") migrated, err := migrate(ctx, bulker, migrateAgentOutputs) if err != nil { return fmt.Errorf("v8.4.0 data migration failed: %w", err) From 9bbd267a141f512d0ddb9d3435acfb99482d9dd7 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Fri, 12 Aug 2022 18:13:50 +0200 Subject: [PATCH 81/89] . --- internal/pkg/dl/migration.go | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 1ab989c29..25112365b 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -136,7 +136,7 @@ func applyMigration(ctx context.Context, name string, index string, bulker bulk. // ============================== V7.15 migration ============================== func migrateTov7_15(ctx context.Context, bulker bulk.Bulk) error { - log.Info().Msg("applying migration to v7.15") + log.Debug().Msg("applying migration to v7.15") _, err := migrate(ctx, bulker, migrateAgentMetadata) if err != nil { return fmt.Errorf("v7.15.0 data migration failed: %w", err) @@ -176,7 +176,7 @@ func migrateAgentMetadata() (string, string, []byte, error) { // https://github.com/elastic/fleet-server/issues/1672 func migrateToV8_4(ctx context.Context, bulker bulk.Bulk) error { - log.Info().Msg("applying migration to v8.4") + log.Debug().Msg("applying migration to v8.4") migrated, err := migrate(ctx, bulker, migrateAgentOutputs) if err != nil { return fmt.Errorf("v8.4.0 data migration failed: %w", err) @@ -209,24 +209,27 @@ func migrateToV8_4(ctx context.Context, bulker bulk.Bulk) error { // However, reverting FleetServer to an older version might cause very issue // this change fixes. func migrateAgentOutputs() (string, string, []byte, error) { - const migrationName = "AgentOutputs" + const ( + migrationName = "AgentOutputs" + fieldOutputs = "outputs" + ) query := dsl.NewRoot() - query.Query().Bool().MustNot().Exists("elasticsearch_outputs") + query.Query().Bool().MustNot().Exists(fieldOutputs) painless := ` // set up the new filed -if (ctx._source['outputs']==null) - {ctx._source['outputs']=new HashMap();} -if (ctx._source['outputs']['default']==null) - {ctx._source['outputs']['default']=new HashMap();} +if (ctx._source['` + fieldOutputs + `']==null) + {ctx._source['` + fieldOutputs + `']=new HashMap();} +if (ctx._source['` + fieldOutputs + `']['default']==null) + {ctx._source['` + fieldOutputs + `']['default']=new HashMap();} // copy old values to new 'outputs' field -ctx._source['outputs']['default'].type="elasticsearch"; -ctx._source['outputs']['default'].to_retire_api_key_ids=ctx._source.default_api_key_history; -ctx._source['outputs']['default'].api_key=ctx._source.default_api_key; -ctx._source['outputs']['default'].api_key_id=ctx._source.default_api_key_id; -ctx._source['outputs']['default'].policy_permissions_hash=ctx._source.policy_output_permissions_hash; +ctx._source['` + fieldOutputs + `']['default'].type="elasticsearch"; +ctx._source['` + fieldOutputs + `']['default'].to_retire_api_key_ids=ctx._source.default_api_key_history; +ctx._source['` + fieldOutputs + `']['default'].api_key=""; +ctx._source['` + fieldOutputs + `']['default'].api_key_id=""; +ctx._source['` + fieldOutputs + `']['default'].policy_permissions_hash=ctx._source.policy_output_permissions_hash; // Erase deprecated fields ctx._source.default_api_key_history=null; From cdac94447feb2fa4c83159d05f4b8a4a981faab5 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 25 Aug 2022 13:43:08 +0200 Subject: [PATCH 82/89] small fixes and PR changes --- internal/pkg/api/handleAck_test.go | 2 +- internal/pkg/bulk/opBulk.go | 2 + internal/pkg/dl/constants.go | 2 +- internal/pkg/dl/migration.go | 39 ++++++++++----- internal/pkg/dl/migration_integration_test.go | 47 ++++++++++++++----- internal/pkg/model/schema.go | 2 +- model/schema.json | 4 +- 7 files changed, 70 insertions(+), 28 deletions(-) diff --git a/internal/pkg/api/handleAck_test.go b/internal/pkg/api/handleAck_test.go index 3613bd5f2..60a265bd4 100644 --- a/internal/pkg/api/handleAck_test.go +++ b/internal/pkg/api/handleAck_test.go @@ -465,7 +465,7 @@ func TestInvalidateAPIKeys(t *testing.T) { bulker := ftesting.NewMockBulk() bulker.On("APIKeyInvalidate", context.Background(), mock.MatchedBy(func(ids []string) bool { - // If A is a subset of B and B is a subset of A => A = B + // if A contains B and B contains A => A = B return assert.Subset(t, ids, want) && assert.Subset(t, want, ids) })). diff --git a/internal/pkg/bulk/opBulk.go b/internal/pkg/bulk/opBulk.go index 650962b86..f20e86c6a 100644 --- a/internal/pkg/bulk/opBulk.go +++ b/internal/pkg/bulk/opBulk.go @@ -226,6 +226,8 @@ func (b *Bulker) flushBulk(ctx context.Context, queue queueT) error { return fmt.Errorf("flushBulk failed, could not unmarshal ES response: %w", err) } if blk.HasErrors { + // We lack information to properly correlate this error with what has failed. + // Thus, for now it'd be mre noise tan information outside an investigation. log.Debug().Err(errors.New(buf.String())).Msg("Bulk call: Es returned an error") } diff --git a/internal/pkg/dl/constants.go b/internal/pkg/dl/constants.go index 037f0f05b..ef7c5567c 100644 --- a/internal/pkg/dl/constants.go +++ b/internal/pkg/dl/constants.go @@ -38,7 +38,7 @@ const ( FieldPolicyID = "policy_id" FieldPolicyOutputAPIKey = "api_key" FieldPolicyOutputAPIKeyID = "api_key_id" - FieldPolicyOutputPermissionsHash = "policy_permissions_hash" + FieldPolicyOutputPermissionsHash = "permissions_hash" FieldPolicyOutputToRetireAPIKeyIDs = "to_retire_api_key_ids" //nolint:gosec // false positive FieldPolicyRevisionIdx = "policy_revision_idx" FieldRevisionIdx = "revision_idx" diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 25112365b..2ba2ac291 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -40,6 +40,9 @@ type ( } ) +// timeNow is used to get the current time. It should be replaced for testing. +var timeNow = time.Now + // Migrate applies, in sequence, the migration functions. Currently, each migration // function is responsible to ensure it only applies the migration if needed, // being a no-op otherwise. @@ -70,9 +73,8 @@ func migrate(ctx context.Context, bulker bulk.Bulk, fn migrationBodyFn) (int, er if resp.VersionConflicts == 0 { break } - - time.Sleep(time.Second) } + return updatedDocs, nil } @@ -210,26 +212,35 @@ func migrateToV8_4(ctx context.Context, bulker bulk.Bulk) error { // this change fixes. func migrateAgentOutputs() (string, string, []byte, error) { const ( - migrationName = "AgentOutputs" - fieldOutputs = "outputs" + migrationName = "AgentOutputs" + fieldOutputs = "outputs" + fieldRetiredAt = "retiredAt" ) query := dsl.NewRoot() query.Query().Bool().MustNot().Exists(fieldOutputs) + fields := map[string]interface{}{fieldRetiredAt: timeNow().UTC().Format(time.RFC3339)} painless := ` -// set up the new filed -if (ctx._source['` + fieldOutputs + `']==null) - {ctx._source['` + fieldOutputs + `']=new HashMap();} -if (ctx._source['` + fieldOutputs + `']['default']==null) - {ctx._source['` + fieldOutputs + `']['default']=new HashMap();} +// set up the new fields +ctx._source['` + fieldOutputs + `']=new HashMap(); +ctx._source['` + fieldOutputs + `']['default']=new HashMap(); +ctx._source['` + fieldOutputs + `']['default'].to_retire_api_key_ids=new ArrayList(); -// copy old values to new 'outputs' field +// copy 'default_api_key_history' to new 'outputs' field ctx._source['` + fieldOutputs + `']['default'].type="elasticsearch"; ctx._source['` + fieldOutputs + `']['default'].to_retire_api_key_ids=ctx._source.default_api_key_history; + +Map map = new HashMap(); +map.put("retired_at", params.` + fieldRetiredAt + `); +map.put("id", ctx._source.default_api_key_id); + +// Make current API key empty, so fleet-server will generate a new one +// Add current API jey to be retired +ctx._source['` + fieldOutputs + `']['default'].to_retire_api_key_ids.add(map); ctx._source['` + fieldOutputs + `']['default'].api_key=""; ctx._source['` + fieldOutputs + `']['default'].api_key_id=""; -ctx._source['` + fieldOutputs + `']['default'].policy_permissions_hash=ctx._source.policy_output_permissions_hash; +ctx._source['` + fieldOutputs + `']['default'].permissions_hash=ctx._source.policy_output_permissions_hash; // Erase deprecated fields ctx._source.default_api_key_history=null; @@ -237,7 +248,11 @@ ctx._source.default_api_key=""; ctx._source.default_api_key_id=""; ctx._source.policy_output_permissions_hash=""; ` - query.Param("script", painless) + query.Param("script", map[string]interface{}{ + "lang": "painless", + "source": painless, + "params": fields, + }) body, err := query.MarshalJSON() if err != nil { diff --git a/internal/pkg/dl/migration_integration_test.go b/internal/pkg/dl/migration_integration_test.go index e70de61a3..cc2ae3f37 100644 --- a/internal/pkg/dl/migration_integration_test.go +++ b/internal/pkg/dl/migration_integration_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/gofrs/uuid" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -22,6 +23,8 @@ import ( ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing" ) +const nowStr = "2022-08-12T16:50:05Z" + func createSomeAgents(t *testing.T, n int, apiKey bulk.APIKey, index string, bulker bulk.Bulk) []string { t.Helper() @@ -33,9 +36,6 @@ func createSomeAgents(t *testing.T, n int, apiKey bulk.APIKey, index string, bul Key: fmt.Sprint(apiKey.Key, i), } - now := time.Now().UTC() - nowStr := now.Format(time.RFC3339) - agentID := uuid.Must(uuid.NewV4()).String() policyID := uuid.Must(uuid.NewV4()).String() @@ -52,7 +52,7 @@ func createSomeAgents(t *testing.T, n int, apiKey bulk.APIKey, index string, bul DefaultAPIKeyHistory: []model.ToRetireAPIKeyIdsItems{ { ID: "old_" + outputAPIKey.ID, - RetiredAt: now.Add(-5 * time.Minute).Format(time.RFC3339), + RetiredAt: nowStr, }, }, } @@ -133,6 +133,12 @@ func TestPolicyCoordinatorIdx(t *testing.T) { } func TestMigrateOutputs(t *testing.T) { + now, err := time.Parse(time.RFC3339, nowStr) + require.NoError(t, err, "could not parse time "+nowStr) + timeNow = func() time.Time { + return now + } + index, bulker := ftesting.SetupCleanIndex(context.Background(), t, FleetAgents) apiKey := bulk.APIKey{ ID: "testAgent_", @@ -148,10 +154,6 @@ func TestMigrateOutputs(t *testing.T) { for i, id := range agentIDs { wantOutputType := "elasticsearch" - wantAPIKey := bulk.APIKey{ - ID: fmt.Sprint(apiKey.ID, i), - Key: fmt.Sprint(apiKey.Key, i), - } got, err := FindAgent( context.Background(), bulker, QueryAgentByID, FieldID, id, WithIndexName(index)) @@ -160,16 +162,39 @@ func TestMigrateOutputs(t *testing.T) { continue } + wantToRetireAPIKeyIds := []model.ToRetireAPIKeyIdsItems{ + { + // Current API should be marked to retire after the migration + ID: fmt.Sprintf("%s%d", apiKey.ID, i), + RetiredAt: timeNow().UTC().Format(time.RFC3339)}, + { + ID: fmt.Sprintf("old_%s%d", apiKey.ID, i), + RetiredAt: nowStr}, + } + // Assert new fields require.Len(t, got.Outputs, 1) - assert.Equal(t, wantAPIKey.Agent(), got.Outputs["default"].APIKey) - assert.Equal(t, wantAPIKey.ID, got.Outputs["default"].APIKeyID) - assert.Equal(t, wantAPIKey.Agent(), got.Outputs["default"].APIKey) + // Default API key is empty to force fleet-server to regenerate them. + assert.Empty(t, got.Outputs["default"].APIKey) + assert.Empty(t, got.Outputs["default"].APIKeyID) + assert.Equal(t, wantOutputType, got.Outputs["default"].Type) assert.Equal(t, fmt.Sprint("a_output_permission_SHA_", i), got.Outputs["default"].PolicyPermissionsHash) + // Assert ToRetireAPIKeyIds contains the expected values, regardless of the order. + for _, want := range wantToRetireAPIKeyIds { + var found bool + for _, got := range got.Outputs["default"].ToRetireAPIKeyIds { + found = found || cmp.Equal(want, got) + } + if !found { + t.Errorf("could not find %#v, in %#v", + want, got.Outputs["default"].ToRetireAPIKeyIds) + } + } + // Assert deprecated fields assert.Empty(t, got.DefaultAPIKey) assert.Empty(t, got.DefaultAPIKey) diff --git a/internal/pkg/model/schema.go b/internal/pkg/model/schema.go index 270d700f6..0d53ad10a 100644 --- a/internal/pkg/model/schema.go +++ b/internal/pkg/model/schema.go @@ -337,7 +337,7 @@ type PolicyOutput struct { APIKeyID string `json:"api_key_id"` // The policy output permissions hash - PolicyPermissionsHash string `json:"policy_permissions_hash"` + PolicyPermissionsHash string `json:"permissions_hash"` // API keys to be invalidated on next agent ack ToRetireAPIKeyIds []ToRetireAPIKeyIdsItems `json:"to_retire_api_key_ids,omitempty"` diff --git a/model/schema.json b/model/schema.json index 9cdd46bed..824ccfc05 100644 --- a/model/schema.json +++ b/model/schema.json @@ -385,7 +385,7 @@ "description": "ID of the API key the Elastic Agent uses to authenticate with elasticsearch", "type": "string" }, - "policy_permissions_hash": { + "permissions_hash": { "description": "The policy output permissions hash", "type": "string" }, @@ -398,7 +398,7 @@ "api_key", "api_key_history", "api_key_id", - "policy_permissions_hash", + "permissions_hash", "type" ] }, From 2966e63ad5e31cf508981d99d1283b6ed99a893f Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Fri, 26 Aug 2022 13:01:59 +0200 Subject: [PATCH 83/89] . --- internal/pkg/model/schema.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/model/schema.go b/internal/pkg/model/schema.go index 0d53ad10a..18abc0883 100644 --- a/internal/pkg/model/schema.go +++ b/internal/pkg/model/schema.go @@ -337,7 +337,7 @@ type PolicyOutput struct { APIKeyID string `json:"api_key_id"` // The policy output permissions hash - PolicyPermissionsHash string `json:"permissions_hash"` + PermissionsHash string `json:"permissions_hash"` // API keys to be invalidated on next agent ack ToRetireAPIKeyIds []ToRetireAPIKeyIdsItems `json:"to_retire_api_key_ids,omitempty"` From ed3f8d4619a941dcc14133d5e53a315a10445ed7 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Fri, 26 Aug 2022 13:26:22 +0200 Subject: [PATCH 84/89] . --- internal/pkg/dl/agent_integration_test.go | 4 +-- internal/pkg/dl/migration_integration_test.go | 2 +- internal/pkg/policy/policy_output.go | 6 ++-- .../policy/policy_output_integration_test.go | 16 +++++----- internal/pkg/policy/policy_output_test.go | 30 +++++++++---------- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/internal/pkg/dl/agent_integration_test.go b/internal/pkg/dl/agent_integration_test.go index 2f8ad9ec1..3baab6c7e 100644 --- a/internal/pkg/dl/agent_integration_test.go +++ b/internal/pkg/dl/agent_integration_test.go @@ -127,8 +127,8 @@ func TestFindAgent_NewModel(t *testing.T) { ID: "TestFindNewModelAgent_APIKeyID_invalidated", RetiredAt: "TestFindNewModelAgent_APIKeyID_invalidated_at"}, }, - APIKeyID: "TestFindNewModelAgent_APIKeyID", - PolicyPermissionsHash: "TestFindNewModelAgent_PolicyPermissionsHash", + APIKeyID: "TestFindNewModelAgent_APIKeyID", + PermissionsHash: "TestFindNewModelAgent_PermisPolicysionsHash", }, } body, err := json.Marshal(model.Agent{ diff --git a/internal/pkg/dl/migration_integration_test.go b/internal/pkg/dl/migration_integration_test.go index cc2ae3f37..916bf478b 100644 --- a/internal/pkg/dl/migration_integration_test.go +++ b/internal/pkg/dl/migration_integration_test.go @@ -181,7 +181,7 @@ func TestMigrateOutputs(t *testing.T) { assert.Equal(t, wantOutputType, got.Outputs["default"].Type) assert.Equal(t, fmt.Sprint("a_output_permission_SHA_", i), - got.Outputs["default"].PolicyPermissionsHash) + got.Outputs["default"].PermissionsHash) // Assert ToRetireAPIKeyIds contains the expected values, regardless of the order. for _, want := range wantToRetireAPIKeyIds { diff --git a/internal/pkg/policy/policy_output.go b/internal/pkg/policy/policy_output.go index f0faceca1..c2728aa1e 100644 --- a/internal/pkg/policy/policy_output.go +++ b/internal/pkg/policy/policy_output.go @@ -95,7 +95,7 @@ func (p *Output) prepareElasticsearch( switch { case output.APIKey == "": zlog.Debug().Msg("must generate api key as default API key is not present") - case p.Role.Sha2 != output.PolicyPermissionsHash: + case p.Role.Sha2 != output.PermissionsHash: // the is actually the OutputPermissionsHash for the default hash. The Agent // document on ES does not have OutputPermissionsHash for any other output // besides the default one. It seems to me error-prone to rely on the default @@ -109,7 +109,7 @@ func (p *Output) prepareElasticsearch( if needNewKey { zlog.Debug(). RawJSON("fleet.policy.roles", p.Role.Raw). - Str("fleet.policy.default.oldHash", output.PolicyPermissionsHash). + Str("fleet.policy.default.oldHash", output.PermissionsHash). Str("fleet.policy.default.newHash", p.Role.Sha2). Msg("Generating a new API key") @@ -123,7 +123,7 @@ func (p *Output) prepareElasticsearch( output.Type = OutputTypeElasticsearch output.APIKey = outputAPIKey.Agent() output.APIKeyID = outputAPIKey.ID - output.PolicyPermissionsHash = p.Role.Sha2 // for the sake of consistency + output.PermissionsHash = p.Role.Sha2 // for the sake of consistency // When a new keys is generated we need to update the Agent record, // this will need to be updated when multiples remote Elasticsearch output diff --git a/internal/pkg/policy/policy_output_integration_test.go b/internal/pkg/policy/policy_output_integration_test.go index 1d301abca..6acd0d9fa 100644 --- a/internal/pkg/policy/policy_output_integration_test.go +++ b/internal/pkg/policy/policy_output_integration_test.go @@ -59,10 +59,10 @@ func TestRenderUpdatePainlessScript(t *testing.T) { wantOutputs := map[string]*model.PolicyOutput{ outputName: { - APIKey: outputAPIKey.Agent(), - APIKeyID: outputAPIKey.ID, - PolicyPermissionsHash: outputPermissionSha, - Type: OutputTypeElasticsearch, + APIKey: outputAPIKey.Agent(), + APIKeyID: outputAPIKey.ID, + PermissionsHash: outputPermissionSha, + Type: OutputTypeElasticsearch, ToRetireAPIKeyIds: append(tt.existingToRetireAPIKeyIds, model.ToRetireAPIKeyIdsItems{ ID: previousAPIKey.ID, RetiredAt: nowStr}), @@ -78,10 +78,10 @@ func TestRenderUpdatePainlessScript(t *testing.T) { EnrolledAt: nowStr, Outputs: map[string]*model.PolicyOutput{ outputName: { - Type: OutputTypeElasticsearch, - APIKey: previousAPIKey.Agent(), - APIKeyID: previousAPIKey.ID, - PolicyPermissionsHash: "old_" + outputPermissionSha, + Type: OutputTypeElasticsearch, + APIKey: previousAPIKey.Agent(), + APIKeyID: previousAPIKey.ID, + PermissionsHash: "old_" + outputPermissionSha, }, }, } diff --git a/internal/pkg/policy/policy_output_test.go b/internal/pkg/policy/policy_output_test.go index beb49a917..d66275d04 100644 --- a/internal/pkg/policy/policy_output_test.go +++ b/internal/pkg/policy/policy_output_test.go @@ -107,12 +107,12 @@ func TestPolicyOutputESPrepare(t *testing.T) { testAgent := &model.Agent{ Outputs: map[string]*model.PolicyOutput{ output.Name: { - ESDocument: model.ESDocument{}, - APIKey: apiKey.Agent(), - ToRetireAPIKeyIds: nil, - APIKeyID: apiKey.ID, - PolicyPermissionsHash: hashPerm, - Type: OutputTypeElasticsearch, + ESDocument: model.ESDocument{}, + APIKey: apiKey.Agent(), + ToRetireAPIKeyIds: nil, + APIKeyID: apiKey.ID, + PermissionsHash: hashPerm, + Type: OutputTypeElasticsearch, }, }, } @@ -128,7 +128,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { assert.Equal(t, apiKey.Agent(), gotOutput.APIKey) assert.Equal(t, apiKey.ID, gotOutput.APIKeyID) - assert.Equal(t, output.Role.Sha2, gotOutput.PolicyPermissionsHash) + assert.Equal(t, output.Role.Sha2, gotOutput.PermissionsHash) assert.Equal(t, output.Type, gotOutput.Type) assert.Empty(t, gotOutput.ToRetireAPIKeyIds) @@ -176,12 +176,12 @@ func TestPolicyOutputESPrepare(t *testing.T) { testAgent := &model.Agent{ Outputs: map[string]*model.PolicyOutput{ output.Name: { - ESDocument: model.ESDocument{}, - APIKey: oldAPIKey.Agent(), - ToRetireAPIKeyIds: nil, - APIKeyID: oldAPIKey.ID, - PolicyPermissionsHash: hashPerm, - Type: OutputTypeElasticsearch, + ESDocument: model.ESDocument{}, + APIKey: oldAPIKey.Agent(), + ToRetireAPIKeyIds: nil, + APIKeyID: oldAPIKey.ID, + PermissionsHash: hashPerm, + Type: OutputTypeElasticsearch, }, }, } @@ -197,7 +197,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { assert.Equal(t, wantAPIKey.Agent(), gotOutput.APIKey) assert.Equal(t, wantAPIKey.ID, gotOutput.APIKeyID) - assert.Equal(t, output.Role.Sha2, gotOutput.PolicyPermissionsHash) + assert.Equal(t, output.Role.Sha2, gotOutput.PermissionsHash) assert.Equal(t, output.Type, gotOutput.Type) // assert.Contains(t, gotOutput.ToRetireAPIKeyIds, oldAPIKey.ID) // TODO: assert on bulker.Update @@ -248,7 +248,7 @@ func TestPolicyOutputESPrepare(t *testing.T) { assert.Equal(t, apiKey.Agent(), gotOutput.APIKey) assert.Equal(t, apiKey.ID, gotOutput.APIKeyID) - assert.Equal(t, output.Role.Sha2, gotOutput.PolicyPermissionsHash) + assert.Equal(t, output.Role.Sha2, gotOutput.PermissionsHash) assert.Equal(t, output.Type, gotOutput.Type) assert.Empty(t, gotOutput.ToRetireAPIKeyIds) From aa4ad32579b3849cdb427b1e376b0a65a8fcb478 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 6 Sep 2022 09:29:41 +0200 Subject: [PATCH 85/89] fix typo --- internal/pkg/bulk/opBulk.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/bulk/opBulk.go b/internal/pkg/bulk/opBulk.go index f20e86c6a..d47ba9592 100644 --- a/internal/pkg/bulk/opBulk.go +++ b/internal/pkg/bulk/opBulk.go @@ -227,7 +227,7 @@ func (b *Bulker) flushBulk(ctx context.Context, queue queueT) error { } if blk.HasErrors { // We lack information to properly correlate this error with what has failed. - // Thus, for now it'd be mre noise tan information outside an investigation. + // Thus, for now it'd be more noise than information outside an investigation. log.Debug().Err(errors.New(buf.String())).Msg("Bulk call: Es returned an error") } From a64a820d208adb6b44c2bfa1a596ad83903cc77e Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 6 Sep 2022 13:54:30 +0200 Subject: [PATCH 86/89] fix linter issues --- internal/pkg/coordinator/monitor_integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/pkg/coordinator/monitor_integration_test.go b/internal/pkg/coordinator/monitor_integration_test.go index 87c30c4ee..190cf05a0 100644 --- a/internal/pkg/coordinator/monitor_integration_test.go +++ b/internal/pkg/coordinator/monitor_integration_test.go @@ -421,10 +421,10 @@ func ensurePolicy(ctx context.Context, t *testing.T, bulker bulk.Bulk, index str if found == nil { //nolint:staticcheck // false positive t.Fatal("policy not found") } - if found.RevisionIdx != revisionIdx { //nolint:staticcheck // found is never nil + if found.RevisionIdx != revisionIdx { //nolint:staticcheck,nolintlint // found is never nil t.Fatal("revision_idx does not match") } - if found.CoordinatorIdx != coordinatorIdx { //nolint:staticcheck // found is never nil + if found.CoordinatorIdx != coordinatorIdx { t.Fatal("coordinator_idx does not match") } } From a3c36d69845d9050bca4c30f26f1cc101f72c572 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 6 Sep 2022 13:56:28 +0200 Subject: [PATCH 87/89] fix linter issues --- internal/pkg/coordinator/monitor_integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/pkg/coordinator/monitor_integration_test.go b/internal/pkg/coordinator/monitor_integration_test.go index 190cf05a0..defc4a9c7 100644 --- a/internal/pkg/coordinator/monitor_integration_test.go +++ b/internal/pkg/coordinator/monitor_integration_test.go @@ -418,10 +418,10 @@ func ensurePolicy(ctx context.Context, t *testing.T, bulker bulk.Bulk, index str break } } - if found == nil { //nolint:staticcheck // false positive + if found == nil { t.Fatal("policy not found") } - if found.RevisionIdx != revisionIdx { //nolint:staticcheck,nolintlint // found is never nil + if found.RevisionIdx != revisionIdx { t.Fatal("revision_idx does not match") } if found.CoordinatorIdx != coordinatorIdx { From 1269b6876ebc05f4cf690d653b8769b105e97bbf Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 6 Sep 2022 22:56:29 +0200 Subject: [PATCH 88/89] adjust migration name --- internal/pkg/dl/migration.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 2ba2ac291..a1857b5aa 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -47,7 +47,7 @@ var timeNow = time.Now // function is responsible to ensure it only applies the migration if needed, // being a no-op otherwise. func Migrate(ctx context.Context, bulker bulk.Bulk) error { - for _, fn := range []migrationFn{migrateTov7_15, migrateToV8_4} { + for _, fn := range []migrationFn{migrateTov7_15, migrateToV8_5} { if err := fn(ctx, bulker); err != nil { return err } @@ -174,14 +174,14 @@ func migrateAgentMetadata() (string, string, []byte, error) { return migrationName, FleetAgents, body, nil } -// ============================== V8.4.0 migration ============================= +// ============================== V8.5.0 migration ============================= // https://github.com/elastic/fleet-server/issues/1672 -func migrateToV8_4(ctx context.Context, bulker bulk.Bulk) error { +func migrateToV8_5(ctx context.Context, bulker bulk.Bulk) error { log.Debug().Msg("applying migration to v8.4") migrated, err := migrate(ctx, bulker, migrateAgentOutputs) if err != nil { - return fmt.Errorf("v8.4.0 data migration failed: %w", err) + return fmt.Errorf("v8.5.0 data migration failed: %w", err) } // The migration was necessary and indeed run, thus we need to regenerate @@ -190,7 +190,7 @@ func migrateToV8_4(ctx context.Context, bulker bulk.Bulk) error { if migrated > 0 { _, err := migrate(ctx, bulker, migratePolicyCoordinatorIdx) if err != nil { - return fmt.Errorf("v8.4.0 data migration failed: %w", err) + return fmt.Errorf("v8.5.0 data migration failed: %w", err) } } @@ -200,7 +200,7 @@ func migrateToV8_4(ctx context.Context, bulker bulk.Bulk) error { // migrateAgentOutputs performs the necessary changes on the Agent documents // to introduce the `Outputs` field. // -// FleetServer 8.4.0 introduces a new field to the Agent document, Outputs, to +// FleetServer 8.5.0 introduces a new field to the Agent document, Outputs, to // store the outputs credentials and data. The DefaultAPIKey, DefaultAPIKeyID, // DefaultAPIKeyHistory and PolicyOutputPermissionsHash are now deprecated in // favour of the new `Outputs` fields, which maps the output name to its data. From b42d5f81991fbbb18184444d1a28ea4f16dcad18 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 6 Sep 2022 22:59:16 +0200 Subject: [PATCH 89/89] adjust migration log --- internal/pkg/dl/migration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index a1857b5aa..5b1722824 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -178,7 +178,7 @@ func migrateAgentMetadata() (string, string, []byte, error) { // https://github.com/elastic/fleet-server/issues/1672 func migrateToV8_5(ctx context.Context, bulker bulk.Bulk) error { - log.Debug().Msg("applying migration to v8.4") + log.Debug().Msg("applying migration to v8.5.0") migrated, err := migrate(ctx, bulker, migrateAgentOutputs) if err != nil { return fmt.Errorf("v8.5.0 data migration failed: %w", err)