Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Updated migration query to match items with deprecated field present #1959

Merged
merged 8 commits into from
Oct 6, 2022
Merged

Updated migration query to match items with deprecated field present #1959

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
e013cd5
updated query
michalpristas Oct 6, 2022
a2753f0
lint
michalpristas Oct 6, 2022
b5580cc
i dont know what i'm doing
michalpristas Oct 6, 2022
5df0c85
i dont know what i'm doing
michalpristas Oct 6, 2022
e392eb7
no template yaay
michalpristas Oct 6, 2022
31fe319
no template yaay
michalpristas Oct 6, 2022
c60482c
set deprecated fields to null and test it
AndersonQ Oct 6, 2022
786b6b4
query for not null DefaultAPIKeyID and ensure it does not migrate twice
AndersonQ Oct 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions internal/pkg/dl/migration.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,13 +217,14 @@ func migrateToV8_5(ctx context.Context, bulker bulk.Bulk) error {
// this change fixes.
func migrateAgentOutputs() (string, string, []byte, error) {
const (
migrationName = "AgentOutputs"
fieldOutputs = "outputs"
fieldRetiredAt = "retiredAt"
migrationName = "AgentOutputs"
fieldOutputs = "outputs"
fieldDefaultAPIKeyID = "default_api_key_id" // nolint:gosec,G101 // this is not a credential
fieldRetiredAt = "retiredAt"
)

query := dsl.NewRoot()
query.Query().Bool().MustNot().Exists(fieldOutputs)
query.Query().Bool().Must().Exists(fieldDefaultAPIKeyID)

fields := map[string]interface{}{fieldRetiredAt: timeNow().UTC().Format(time.RFC3339)}
painless := `
Expand Down Expand Up @@ -253,9 +254,9 @@ ctx._source['` + fieldOutputs + `']['default'].permissions_hash=ctx._source.poli

// Erase deprecated fields
ctx._source.default_api_key_history=null;
ctx._source.default_api_key="";
ctx._source.default_api_key_id="";
ctx._source.policy_output_permissions_hash="";
ctx._source.default_api_key=null;
ctx._source.default_api_key_id=null;
ctx._source.policy_output_permissions_hash=null;
`
query.Param("script", map[string]interface{}{
"lang": "painless",
Expand Down
97 changes: 80 additions & 17 deletions internal/pkg/dl/migration_integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,12 +155,29 @@ func TestMigrateOutputs_withDefaultAPIKeyHistory(t *testing.T) {
for i, id := range agentIDs {
wantOutputType := "elasticsearch" //nolint:goconst // test cases have some duplication

got, err := FindAgent(
context.Background(), bulker, QueryAgentByID, FieldID, id, WithIndexName(index))
if err != nil {
assert.NoError(t, err, "failed to find agent ID %q", id) // we want to continue even if a single agent fails
continue
}
res, err := SearchWithOneParam(context.Background(), bulker, QueryAgentByID, index, FieldID, id)
require.NoError(t, err)
require.Len(t, res.Hits, 1)

var got model.Agent
err = res.Hits[0].Unmarshal(&got)
require.NoError(t, err, "could not unmarshal ES document into model.Agent")

gotDeprecatedFields := struct {
// Deprecated. Use Outputs instead. API key the Elastic Agent uses to authenticate with elasticsearch
DefaultAPIKey *string `json:"default_api_key,omitempty"`

// Deprecated. Use Outputs instead. Default API Key History
DefaultAPIKeyHistory []model.ToRetireAPIKeyIdsItems `json:"default_api_key_history,omitempty"`

// Deprecated. Use Outputs instead. ID of the API key the Elastic Agent uses to authenticate with elasticsearch
DefaultAPIKeyID *string `json:"default_api_key_id,omitempty"`

// Deprecated. Use Outputs instead. The policy output permissions hash
PolicyOutputPermissionsHash *string `json:"policy_output_permissions_hash,omitempty"`
}{}
err = res.Hits[0].Unmarshal(&gotDeprecatedFields)
require.NoError(t, err, "could not unmarshal ES document into gotDeprecatedFields")

wantToRetireAPIKeyIds := []model.ToRetireAPIKeyIdsItems{
{
Expand Down Expand Up @@ -196,11 +213,36 @@ func TestMigrateOutputs_withDefaultAPIKeyHistory(t *testing.T) {
}

// Assert deprecated fields
assert.Empty(t, got.DefaultAPIKey)
assert.Empty(t, got.DefaultAPIKey)
assert.Empty(t, got.PolicyOutputPermissionsHash)
assert.Nil(t, got.DefaultAPIKeyHistory)
assert.Nil(t, gotDeprecatedFields.DefaultAPIKey)
assert.Nil(t, gotDeprecatedFields.DefaultAPIKeyID)
assert.Nil(t, gotDeprecatedFields.PolicyOutputPermissionsHash)
assert.Nil(t, gotDeprecatedFields.DefaultAPIKeyHistory)
}
}

func TestMigrateOutputs_dontMigrateTwice(t *testing.T) {
now, err := time.Parse(time.RFC3339, nowStr)
require.NoError(t, err, "could not parse time "+nowStr)
timeNow = func() time.Time {
return now
}

index, bulker := ftesting.SetupCleanIndex(context.Background(), t, FleetAgents)
apiKey := bulk.APIKey{
ID: "testAgent_",
Key: "testAgent_key_",
}

agentIDs := createSomeAgents(t, 25, apiKey, index, bulker)

migratedAgents, err := migrate(context.Background(), bulker, migrateAgentOutputs)
require.NoError(t, err)
assert.Equal(t, len(agentIDs), migratedAgents)

migratedAgents2, err := migrate(context.Background(), bulker, migrateAgentOutputs)
require.NoError(t, err)

assert.Equal(t, 0, migratedAgents2)
}

func TestMigrateOutputs_nil_DefaultAPIKeyHistory(t *testing.T) {
Expand Down Expand Up @@ -249,9 +291,30 @@ func TestMigrateOutputs_nil_DefaultAPIKeyHistory(t *testing.T) {
migratedAgents, err := migrate(context.Background(), bulker, migrateAgentOutputs)
require.NoError(t, err)

got, err := FindAgent(
context.Background(), bulker, QueryAgentByID, FieldID, agentID, WithIndexName(index))
require.NoError(t, err, "failed to find agent ID %q", agentID) // we want to continue even if a single agent fails
res, err := SearchWithOneParam(
context.Background(), bulker, QueryAgentByID, index, FieldID, agentID)
require.NoError(t, err, "failed to find agent ID %q", agentID)
require.Len(t, res.Hits, 1)

var got model.Agent
err = res.Hits[0].Unmarshal(&got)
require.NoError(t, err, "could not unmarshal ES document into model.Agent")

gotDeprecatedFields := struct {
// Deprecated. Use Outputs instead. API key the Elastic Agent uses to authenticate with elasticsearch
DefaultAPIKey *string `json:"default_api_key,omitempty"`

// Deprecated. Use Outputs instead. Default API Key History
DefaultAPIKeyHistory []model.ToRetireAPIKeyIdsItems `json:"default_api_key_history,omitempty"`

// Deprecated. Use Outputs instead. ID of the API key the Elastic Agent uses to authenticate with elasticsearch
DefaultAPIKeyID *string `json:"default_api_key_id,omitempty"`

// Deprecated. Use Outputs instead. The policy output permissions hash
PolicyOutputPermissionsHash *string `json:"policy_output_permissions_hash,omitempty"`
}{}
err = res.Hits[0].Unmarshal(&gotDeprecatedFields)
require.NoError(t, err, "could not unmarshal ES document into gotDeprecatedFields")

assert.Equal(t, 1, migratedAgents)

Expand All @@ -273,10 +336,10 @@ func TestMigrateOutputs_nil_DefaultAPIKeyHistory(t *testing.T) {
}

// Assert deprecated fields
assert.Empty(t, got.DefaultAPIKey)
assert.Empty(t, got.DefaultAPIKey)
assert.Empty(t, got.PolicyOutputPermissionsHash)
assert.Nil(t, got.DefaultAPIKeyHistory)
assert.Nil(t, gotDeprecatedFields.DefaultAPIKey)
assert.Nil(t, gotDeprecatedFields.DefaultAPIKey)
assert.Nil(t, gotDeprecatedFields.PolicyOutputPermissionsHash)
assert.Nil(t, gotDeprecatedFields.DefaultAPIKeyHistory)
}

func TestMigrateOutputs_no_agent_document(t *testing.T) {
Expand Down