diff --git a/cmd/esmapping-generator/app/flags.go b/cmd/esmapping-generator/app/flags.go index 6d2590fd5bf..154ff2cfa41 100644 --- a/cmd/esmapping-generator/app/flags.go +++ b/cmd/esmapping-generator/app/flags.go @@ -20,21 +20,23 @@ import ( // Options represent configurable parameters for jaeger-esmapping-generator type Options struct { - Mapping string - EsVersion uint - Shards int64 - Replicas int64 - IndexPrefix string - UseILM string // using string as util is being used in python and using bool leads to type issues. + Mapping string + EsVersion uint + Shards int64 + Replicas int64 + IndexPrefix string + UseILM string // using string as util is being used in python and using bool leads to type issues. + ILMPolicyName string } const ( - mappingFlag = "mapping" - esVersionFlag = "es-version" - shardsFlag = "shards" - replicasFlag = "replicas" - indexPrefixFlag = "index-prefix" - useILMFlag = "use-ilm" + mappingFlag = "mapping" + esVersionFlag = "es-version" + shardsFlag = "shards" + replicasFlag = "replicas" + indexPrefixFlag = "index-prefix" + useILMFlag = "use-ilm" + ilmPolicyNameFlag = "ilm-policy-name" ) // AddFlags adds flags for esmapping-generator main program @@ -69,6 +71,11 @@ func (o *Options) AddFlags(command *cobra.Command) { useILMFlag, "false", "Set to true to use ILM for managing lifecycle of jaeger indices") + command.Flags().StringVar( + &o.ILMPolicyName, + ilmPolicyNameFlag, + "jaeger-ilm-policy", + "The name of the ILM policy to use if ILM is active") // mark mapping flag as mandatory command.MarkFlagRequired(mappingFlag) diff --git a/cmd/esmapping-generator/app/flags_test.go b/cmd/esmapping-generator/app/flags_test.go index ca66a0ff649..6f4410b2596 100644 --- a/cmd/esmapping-generator/app/flags_test.go +++ b/cmd/esmapping-generator/app/flags_test.go @@ -33,6 +33,7 @@ func TestOptionsWithDefaultFlags(t *testing.T) { assert.Equal(t, int64(1), o.Replicas) assert.Equal(t, "", o.IndexPrefix) assert.Equal(t, "false", o.UseILM) + assert.Equal(t, "jaeger-ilm-policy", o.ILMPolicyName) } func TestOptionsWithFlags(t *testing.T) { @@ -47,6 +48,7 @@ func TestOptionsWithFlags(t *testing.T) { "--replicas=1", "--index-prefix=test", "--use-ilm=true", + "--ilm-policy-name=jaeger-test-policy", }) require.NoError(t, err) assert.Equal(t, "jaeger-span", o.Mapping) @@ -55,4 +57,5 @@ func TestOptionsWithFlags(t *testing.T) { assert.Equal(t, int64(1), o.Replicas) assert.Equal(t, "test", o.IndexPrefix) assert.Equal(t, "true", o.UseILM) + assert.Equal(t, "jaeger-test-policy", o.ILMPolicyName) } diff --git a/cmd/esmapping-generator/app/renderer/render.go b/cmd/esmapping-generator/app/renderer/render.go index 2e96703f37b..993ab5bc12e 100644 --- a/cmd/esmapping-generator/app/renderer/render.go +++ b/cmd/esmapping-generator/app/renderer/render.go @@ -42,6 +42,7 @@ func GetMappingAsString(builder es.TemplateBuilder, opt *app.Options) (string, e EsVersion: opt.EsVersion, IndexPrefix: opt.IndexPrefix, UseILM: enableILM, + ILMPolicyName: opt.ILMPolicyName, } return mappingBuilder.GetMapping(opt.Mapping) } diff --git a/cmd/esmapping-generator/app/renderer/render_test.go b/cmd/esmapping-generator/app/renderer/render_test.go index e6130bd80e7..6ba937909ec 100644 --- a/cmd/esmapping-generator/app/renderer/render_test.go +++ b/cmd/esmapping-generator/app/renderer/render_test.go @@ -50,23 +50,23 @@ func Test_getMappingAsString(t *testing.T) { wantErr error }{ { - name: "ES version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "true"}, + name: "ES version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "true", ILMPolicyName: "jaeger-test-policy"}, want: "ES version 7", }, { - name: "ES version 6", args: app.Options{Mapping: "jaeger-span", EsVersion: 6, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "false"}, + name: "ES version 6", args: app.Options{Mapping: "jaeger-span", EsVersion: 6, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "false", ILMPolicyName: "jaeger-test-policy"}, want: "ES version 6", }, { - name: "Parse Error version 6", args: app.Options{Mapping: "jaeger-span", EsVersion: 6, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "false"}, + name: "Parse Error version 6", args: app.Options{Mapping: "jaeger-span", EsVersion: 6, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "false", ILMPolicyName: "jaeger-test-policy"}, wantErr: errors.New("parse error"), }, { - name: "Parse Error version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "true"}, + name: "Parse Error version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "true", ILMPolicyName: "jaeger-test-policy"}, wantErr: errors.New("parse error"), }, { - name: "Parse bool error", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "foo"}, + name: "Parse bool error", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "foo", ILMPolicyName: "jaeger-test-policy"}, wantErr: errors.New("strconv.ParseBool: parsing \"foo\": invalid syntax"), }, } diff --git a/plugin/storage/es/esRollover.py b/plugin/storage/es/esRollover.py index 5697c2b7c8c..36c9486e9ed 100755 --- a/plugin/storage/es/esRollover.py +++ b/plugin/storage/es/esRollover.py @@ -18,6 +18,7 @@ UNIT_COUNT = 2 SHARDS = 5 REPLICAS = 1 +ILM_POLICY_NAME = 'jaeger-ilm-policy' def main(): @@ -39,6 +40,7 @@ def main(): print('ES_TLS_CERT ... Path to TLS certificate file.') print('ES_TLS_KEY ... Path to TLS key file.') print('ES_USE_ILM .. Use ILM to manage jaeger indices.') + print('ES_ILM_POLICY_NAME .. The name of the ILM policy to use if ILM is active.') print('ES_TLS_SKIP_HOST_VERIFY ... (insecure) Skip server\'s certificate chain and host name verification.') print( 'ES_VERSION ... The major Elasticsearch version. If not specified, the value will be auto-detected from Elasticsearch.') @@ -83,13 +85,14 @@ def perform_action(action, client, write_alias, read_alias, index_to_rollover, t replicas = os.getenv('REPLICAS', REPLICAS) esVersion = get_version(client) use_ilm = str2bool(os.getenv("ES_USE_ILM", 'false')) + ilm_policy_name = os.getenv('ES_ILM_POLICY_NAME', ILM_POLICY_NAME) if esVersion == 7: if use_ilm: - check_if_ilm_policy_exists("jaeger-ilm-policy") + check_if_ilm_policy_exists(ilm_policy_name) else: if use_ilm: sys.exit("ILM is supported only for ES version 7+") - create_index_template(fix_mapping(template_name, esVersion, shards, replicas, prefix.rstrip("-"), use_ilm), + create_index_template(fix_mapping(template_name, esVersion, shards, replicas, prefix.rstrip("-"), use_ilm, ilm_policy_name), prefix + template_name) index = index_to_rollover + '-000001' @@ -194,10 +197,11 @@ def str2bool(v): return v.lower() in ('true', '1') -def fix_mapping(template_name, esVersion, shards, replicas, indexPrefix, use_ilm): +def fix_mapping(template_name, esVersion, shards, replicas, indexPrefix, use_ilm, ilm_policy_name): output = subprocess.Popen(['esmapping-generator', '--mapping', template_name, '--es-version', str(esVersion), '--shards', str(shards), '--replicas', - str(replicas), '--index-prefix', indexPrefix, '--use-ilm', str(use_ilm)], + str(replicas), '--index-prefix', indexPrefix, + '--use-ilm', str(use_ilm), '--ilm-policy-name', ilm_policy_name], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) mapping, stderr = output.communicate() diff --git a/plugin/storage/es/mappings/fixtures/jaeger-service-7.json b/plugin/storage/es/mappings/fixtures/jaeger-service-7.json index 6f1ad6697b5..c17574456e5 100644 --- a/plugin/storage/es/mappings/fixtures/jaeger-service-7.json +++ b/plugin/storage/es/mappings/fixtures/jaeger-service-7.json @@ -9,7 +9,7 @@ "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true ,"lifecycle": { - "name": "jaeger-ilm-policy", + "name": "jaeger-test-policy", "rollover_alias": "test-jaeger-service-write" } }, diff --git a/plugin/storage/es/mappings/fixtures/jaeger-span-7.json b/plugin/storage/es/mappings/fixtures/jaeger-span-7.json index 590cf6172af..ec002507a24 100644 --- a/plugin/storage/es/mappings/fixtures/jaeger-span-7.json +++ b/plugin/storage/es/mappings/fixtures/jaeger-span-7.json @@ -9,7 +9,7 @@ "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true ,"lifecycle": { - "name": "jaeger-ilm-policy", + "name": "jaeger-test-policy", "rollover_alias": "test-jaeger-span-write" } }, diff --git a/plugin/storage/es/mappings/jaeger-service-7.json b/plugin/storage/es/mappings/jaeger-service-7.json index 2dd1eb63333..0ca2d186319 100644 --- a/plugin/storage/es/mappings/jaeger-service-7.json +++ b/plugin/storage/es/mappings/jaeger-service-7.json @@ -12,7 +12,7 @@ "index.requests.cache.enable":true {{- if .UseILM }} ,"lifecycle": { - "name": "jaeger-ilm-policy", + "name": "{{ .ILMPolicyName }}", "rollover_alias": "{{ .IndexPrefix }}jaeger-service-write" } {{- end }} diff --git a/plugin/storage/es/mappings/jaeger-span-7.json b/plugin/storage/es/mappings/jaeger-span-7.json index 79a495b1c16..3d8bbae95dc 100644 --- a/plugin/storage/es/mappings/jaeger-span-7.json +++ b/plugin/storage/es/mappings/jaeger-span-7.json @@ -12,7 +12,7 @@ "index.requests.cache.enable":true {{- if .UseILM }} ,"lifecycle": { - "name": "jaeger-ilm-policy", + "name": "{{ .ILMPolicyName }}", "rollover_alias": "{{ .IndexPrefix }}jaeger-span-write" } {{- end }} diff --git a/plugin/storage/es/mappings/mapping.go b/plugin/storage/es/mappings/mapping.go index f055217bbb7..418a692edbf 100644 --- a/plugin/storage/es/mappings/mapping.go +++ b/plugin/storage/es/mappings/mapping.go @@ -34,6 +34,7 @@ type MappingBuilder struct { EsVersion uint IndexPrefix string UseILM bool + ILMPolicyName string } // GetMapping returns the rendered mapping based on elasticsearch version diff --git a/plugin/storage/es/mappings/mapping_test.go b/plugin/storage/es/mappings/mapping_test.go index 683f68f4d5b..a47a5975c24 100644 --- a/plugin/storage/es/mappings/mapping_test.go +++ b/plugin/storage/es/mappings/mapping_test.go @@ -54,6 +54,7 @@ func TestMappingBuilder_GetMapping(t *testing.T) { EsVersion: tt.esVersion, IndexPrefix: "test-", UseILM: true, + ILMPolicyName: "jaeger-test-policy", } got, err := mb.GetMapping(tt.mapping) require.NoError(t, err) @@ -142,6 +143,7 @@ func TestMappingBuilder_fixMapping(t *testing.T) { EsVersion: 7, IndexPrefix: "test", UseILM: true, + ILMPolicyName: "jaeger-test-policy", } _, err := mappingBuilder.fixMapping("test") if test.err != "" { @@ -156,11 +158,12 @@ func TestMappingBuilder_fixMapping(t *testing.T) { func TestMappingBuilder_GetSpanServiceMappings(t *testing.T) { type args struct { - shards int64 - replicas int64 - esVersion uint - indexPrefix string - useILM bool + shards int64 + replicas int64 + esVersion uint + indexPrefix string + useILM bool + ilmPolicyName string } tests := []struct { name string @@ -171,11 +174,12 @@ func TestMappingBuilder_GetSpanServiceMappings(t *testing.T) { { name: "ES Version 7", args: args{ - shards: 3, - replicas: 3, - esVersion: 7, - indexPrefix: "test", - useILM: true, + shards: 3, + replicas: 3, + esVersion: 7, + indexPrefix: "test", + useILM: true, + ilmPolicyName: "jaeger-test-policy", }, mockNewTextTemplateBuilder: func() es.TemplateBuilder { tb := mocks.TemplateBuilder{} @@ -189,11 +193,12 @@ func TestMappingBuilder_GetSpanServiceMappings(t *testing.T) { { name: "ES Version 7 Service Error", args: args{ - shards: 3, - replicas: 3, - esVersion: 7, - indexPrefix: "test", - useILM: true, + shards: 3, + replicas: 3, + esVersion: 7, + indexPrefix: "test", + useILM: true, + ilmPolicyName: "jaeger-test-policy", }, mockNewTextTemplateBuilder: func() es.TemplateBuilder { tb := mocks.TemplateBuilder{} @@ -209,11 +214,12 @@ func TestMappingBuilder_GetSpanServiceMappings(t *testing.T) { { name: "ES Version < 7", args: args{ - shards: 3, - replicas: 3, - esVersion: 6, - indexPrefix: "test", - useILM: true, + shards: 3, + replicas: 3, + esVersion: 6, + indexPrefix: "test", + useILM: true, + ilmPolicyName: "jaeger-test-policy", }, mockNewTextTemplateBuilder: func() es.TemplateBuilder { tb := mocks.TemplateBuilder{} @@ -227,11 +233,12 @@ func TestMappingBuilder_GetSpanServiceMappings(t *testing.T) { { name: "ES Version < 7 Service Error", args: args{ - shards: 3, - replicas: 3, - esVersion: 6, - indexPrefix: "test", - useILM: true, + shards: 3, + replicas: 3, + esVersion: 6, + indexPrefix: "test", + useILM: true, + ilmPolicyName: "jaeger-test-policy", }, mockNewTextTemplateBuilder: func() es.TemplateBuilder { tb := mocks.TemplateBuilder{} @@ -246,11 +253,12 @@ func TestMappingBuilder_GetSpanServiceMappings(t *testing.T) { { name: "ES Version < 7 Span Error", args: args{ - shards: 3, - replicas: 3, - esVersion: 6, - indexPrefix: "test", - useILM: true, + shards: 3, + replicas: 3, + esVersion: 6, + indexPrefix: "test", + useILM: true, + ilmPolicyName: "jaeger-test-policy", }, mockNewTextTemplateBuilder: func() es.TemplateBuilder { tb := mocks.TemplateBuilder{} @@ -264,11 +272,12 @@ func TestMappingBuilder_GetSpanServiceMappings(t *testing.T) { { name: "ES Version 7 Span Error", args: args{ - shards: 3, - replicas: 3, - esVersion: 7, - indexPrefix: "test", - useILM: true, + shards: 3, + replicas: 3, + esVersion: 7, + indexPrefix: "test", + useILM: true, + ilmPolicyName: "jaeger-test-policy", }, mockNewTextTemplateBuilder: func() es.TemplateBuilder { tb := mocks.TemplateBuilder{} @@ -289,6 +298,7 @@ func TestMappingBuilder_GetSpanServiceMappings(t *testing.T) { EsVersion: test.args.esVersion, IndexPrefix: test.args.indexPrefix, UseILM: test.args.useILM, + ILMPolicyName: test.args.ilmPolicyName, } _, _, err := mappingBuilder.GetSpanServiceMappings() if test.err != "" { diff --git a/plugin/storage/integration/es_index_rollover_test.go b/plugin/storage/integration/es_index_rollover_test.go index 5a3f7f34896..84a77445662 100644 --- a/plugin/storage/integration/es_index_rollover_test.go +++ b/plugin/storage/integration/es_index_rollover_test.go @@ -29,8 +29,8 @@ import ( ) const ( - indexILMName = "jaeger-ilm-policy" - rolloverImage = "jaegertracing/jaeger-es-rollover:latest" + defaultILMPolicyName = "jaeger-ilm-policy" + rolloverImage = "jaegertracing/jaeger-es-rollover:latest" ) func TestIndexRollover_FailIfILMNotPresent(t *testing.T) { @@ -42,7 +42,7 @@ func TestIndexRollover_FailIfILMNotPresent(t *testing.T) { t.Skip("Integration test - " + t.Name() + " against ElasticSearch skipped for ES version " + fmt.Sprint(esVersion)) } // make sure ES is clean - cleanES(t, client, "jaeger-ilm-policy") + cleanES(t, client, defaultILMPolicyName) envVars := []string{"ES_USE_ILM=true"} err = runEsRollover("init", envVars) assert.EqualError(t, err, "exit status 1") @@ -52,40 +52,61 @@ func TestIndexRollover_FailIfILMNotPresent(t *testing.T) { } func TestIndexRollover_CreateIndicesWithILM(t *testing.T) { + + // Test using the default ILM Policy Name, i.e. do not pass the ES_ILM_POLICY_NAME env var to the rollover script. + t.Run(fmt.Sprintf("DefaultPolicyName"), func(t *testing.T) { + runCreateIndicesWithILM(t, defaultILMPolicyName) + }) + + // Test using a configured ILM Policy Name, i.e. pass the ES_ILM_POLICY_NAME env var to the rollover script. + t.Run(fmt.Sprintf("SetPolicyName"), func(t *testing.T) { + runCreateIndicesWithILM(t, "jaeger-test-policy") + }) +} + +func runCreateIndicesWithILM(t *testing.T, ilmPolicyName string) { + client, err := createESClient() require.NoError(t, err) esVersion, err := getVersion(client) require.NoError(t, err) + envVars := []string{ + "ES_USE_ILM=true", + } + + if ilmPolicyName != defaultILMPolicyName { + envVars = append(envVars, "ES_ILM_POLICY_NAME="+ilmPolicyName) + } + if esVersion != 7 { cleanES(t, client, "") - err := runEsRollover("init", []string{"ES_USE_ILM=true"}) + err := runEsRollover("init", envVars) assert.EqualError(t, err, "exit status 1") indices, err1 := client.IndexNames() require.NoError(t, err1) assert.Empty(t, indices) } else { - envVars := []string{"ES_USE_ILM=true"} expectedIndices := []string{"jaeger-span-000001", "jaeger-service-000001"} - t.Run(fmt.Sprintf("%s_no_prefix", "CreateIndicesWithILM"), func(t *testing.T) { - runIndexRolloverWithILMTest(t, client, "", expectedIndices, envVars) + t.Run(fmt.Sprintf("NoPrefix"), func(t *testing.T) { + runIndexRolloverWithILMTest(t, client, "", expectedIndices, envVars, ilmPolicyName) }) - t.Run(fmt.Sprintf("%s_prefix", "CreateIndicesWithILM"), func(t *testing.T) { - runIndexRolloverWithILMTest(t, client, indexPrefix, expectedIndices, append(envVars, "INDEX_PREFIX="+indexPrefix)) + t.Run(fmt.Sprintf("WithPrefix"), func(t *testing.T) { + runIndexRolloverWithILMTest(t, client, indexPrefix, expectedIndices, append(envVars, "INDEX_PREFIX="+indexPrefix), ilmPolicyName) }) } } -func runIndexRolloverWithILMTest(t *testing.T, client *elastic.Client, prefix string, expectedIndices, envVars []string) { +func runIndexRolloverWithILMTest(t *testing.T, client *elastic.Client, prefix string, expectedIndices, envVars []string, ilmPolicyName string) { writeAliases := []string{"jaeger-service-write", "jaeger-span-write"} // make sure ES is cleaned before test - cleanES(t, client, "jaeger-ilm-policy") + cleanES(t, client, ilmPolicyName) // make sure ES is cleaned after test - defer cleanES(t, client, "jaeger-ilm-policy") - err := createILMPolicy(client, "jaeger-ilm-policy") + defer cleanES(t, client, ilmPolicyName) + err := createILMPolicy(client, ilmPolicyName) require.NoError(t, err) if prefix != "" { @@ -111,7 +132,7 @@ func runIndexRolloverWithILMTest(t *testing.T, client *elastic.Client, prefix st require.NoError(t, err) //Check ILM Policy is attached and Get rollover alias attached for _, v := range settings { - assert.Equal(t, indexILMName, v.Settings["index.lifecycle.name"]) + assert.Equal(t, ilmPolicyName, v.Settings["index.lifecycle.name"]) actualWriteAliases = append(actualWriteAliases, v.Settings["index.lifecycle.rollover_alias"].(string)) } //Check indices created