Skip to content

Commit 39a8643

Browse files
pugnascotiapugnascotia
authored
Change deprecation indexing to use a custom template (#64417)
The implementation for indexing deprecation logs to a data stream (#58924) relied on the Stack template for `logs-*-*`. This meant that if the user disabled the stack templates, the templates would also be unavailable for the deprecation logs. Change the implementation so that: * There is a separate template for deprecation logging * The data stream is marked as hidden * The data stream name is prefixed with a period (`.`)
1 parent 1990e24 commit 39a8643

File tree

10 files changed

+211
-6
lines changed

10 files changed

+211
-6
lines changed

‎test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1313,6 +1313,7 @@ protected static boolean isXPackTemplate(String name) {
13131313
case "synthetics-settings":
13141314
case "synthetics-mappings":
13151315
case ".snapshot-blob-cache":
1316+
case ".deprecation-indexing-template":
13161317
return true;
13171318
default:
13181319
return false;

‎x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-ilm-policy.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"phases": {
3+
"hot": {
4+
"actions": {
5+
"rollover": {
6+
"max_size": "50gb",
7+
"max_age": "30d"
8+
}
9+
}
10+
}
11+
}
12+
}

‎x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
{
2+
"template": {
3+
"mappings": {
4+
"dynamic_templates": [
5+
{
6+
"strings_as_keyword": {
7+
"mapping": {
8+
"ignore_above": 1024,
9+
"type": "keyword"
10+
},
11+
"match_mapping_type": "string"
12+
}
13+
}
14+
],
15+
"date_detection": false,
16+
"properties": {
17+
"@timestamp": {
18+
"type": "date"
19+
},
20+
"data_stream": {
21+
"properties": {
22+
"type": {
23+
"type": "constant_keyword",
24+
"value": "logs"
25+
},
26+
"dataset": {
27+
"type": "constant_keyword"
28+
},
29+
"namespace": {
30+
"type": "constant_keyword"
31+
}
32+
}
33+
},
34+
"ecs": {
35+
"properties": {
36+
"version": {
37+
"ignore_above": 1024,
38+
"type": "keyword"
39+
}
40+
}
41+
},
42+
"host": {
43+
"properties": {
44+
"ip": {
45+
"type": "ip"
46+
}
47+
}
48+
},
49+
"message": {
50+
"type": "text"
51+
}
52+
}
53+
}
54+
},
55+
"_meta": {
56+
"description": "default mappings for ES deprecation logs index template installed by x-pack",
57+
"managed": true
58+
},
59+
"version": ${xpack.deprecation.indexing.template.version}
60+
}

‎x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"template": {
3+
"settings": {
4+
"index": {
5+
"lifecycle": {
6+
"name": ".deprecation-indexing-ilm-policy"
7+
},
8+
"codec": "best_compression",
9+
"query": {
10+
"default_field": ["message"]
11+
}
12+
}
13+
}
14+
},
15+
"_meta": {
16+
"description": "default settings for ES deprecation logs index template installed by x-pack",
17+
"managed": true
18+
},
19+
"version": ${xpack.deprecation.indexing.template.version}
20+
}

‎x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"index_patterns": [".logs-deprecation-elasticsearch"],
3+
"priority": 1000,
4+
"data_stream": {
5+
"hidden": true
6+
},
7+
"composed_of": [
8+
".deprecation-indexing-mappings",
9+
".deprecation-indexing-settings"
10+
],
11+
"allow_auto_create": true,
12+
"_meta": {
13+
"description": "default template for ES deprecation logs index template installed by x-pack",
14+
"managed": true
15+
},
16+
"version": ${xpack.deprecation.indexing.template.version}
17+
}

‎x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,7 @@ public void testDeprecationMessagesCanBeIndexed() throws Exception {
238238
assertBusy(() -> {
239239
Response response;
240240
try {
241-
response = client().performRequest(new Request("GET", "logs-deprecation-elasticsearch/_search"));
241+
response = client().performRequest(new Request("GET", ".logs-deprecation-elasticsearch/_search"));
242242
} catch (Exception e) {
243243
// It can take a moment for the index to be created. If it doesn't exist then the client
244244
// throws an exception. Translate it into an assertion error so that assertBusy() will
@@ -307,7 +307,7 @@ public void testDeprecationMessagesCanBeIndexed() throws Exception {
307307
});
308308
} finally {
309309
configureWriteDeprecationLogsToIndex(null);
310-
client().performRequest(new Request("DELETE", "_data_stream/logs-deprecation-elasticsearch"));
310+
client().performRequest(new Request("DELETE", "_data_stream/.logs-deprecation-elasticsearch"));
311311
}
312312
}
313313

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/Deprecation.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
3232
import org.elasticsearch.xpack.core.deprecation.NodesDeprecationCheckAction;
3333
import org.elasticsearch.xpack.deprecation.logging.DeprecationIndexingComponent;
34+
import org.elasticsearch.xpack.deprecation.logging.DeprecationIndexingTemplateRegistry;
3435

3536
import java.util.Collection;
3637
import java.util.Collections;
@@ -75,8 +76,11 @@ public Collection<Object> createComponents(
7576
IndexNameExpressionResolver indexNameExpressionResolver,
7677
Supplier<RepositoriesService> repositoriesServiceSupplier
7778
) {
78-
DeprecationIndexingComponent component = new DeprecationIndexingComponent(client, environment.settings());
79+
final DeprecationIndexingTemplateRegistry templateRegistry =
80+
new DeprecationIndexingTemplateRegistry(environment.settings(), clusterService, threadPool, client, xContentRegistry);
81+
templateRegistry.initialize();
7982

83+
final DeprecationIndexingComponent component = new DeprecationIndexingComponent(client, environment.settings());
8084
clusterService.addListener(component);
8185

8286
return List.of(component);

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingAppender.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
*/
2828
@Plugin(name = "DeprecationIndexingAppender", category = Core.CATEGORY_NAME, elementType = Appender.ELEMENT_TYPE)
2929
public class DeprecationIndexingAppender extends AbstractAppender {
30-
public static final String DEPRECATION_MESSAGES_DATA_STREAM = "logs-deprecation-elasticsearch";
30+
public static final String DEPRECATION_MESSAGES_DATA_STREAM = ".logs-deprecation-elasticsearch";
3131

3232
private final Consumer<IndexRequest> requestConsumer;
3333

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingComponent.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,13 +116,12 @@ public void clusterChanged(ClusterChangedEvent event) {
116116
* @return an initialised bulk processor
117117
*/
118118
private BulkProcessor getBulkProcessor(Client client, Settings settings) {
119-
final OriginSettingClient originSettingClient = new OriginSettingClient(client, ClientHelper.DEPRECATION_ORIGIN);
120119
final BulkProcessor.Listener listener = new DeprecationBulkListener();
121120

122121
// This configuration disables the size count and size thresholds,
123122
// and instead uses a scheduled flush only. This means that calling
124123
// processor.add() will not block the calling thread.
125-
return BulkProcessor.builder(originSettingClient::bulk, listener)
124+
return BulkProcessor.builder(client::bulk, listener)
126125
.setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(1000), 3))
127126
.setConcurrentRequests(Math.max(2, EsExecutors.allocatedProcessors(settings)))
128127
.setBulkActions(-1)

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingTemplateRegistry.java‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.deprecation.logging;
8+
9+
import org.elasticsearch.client.Client;
10+
import org.elasticsearch.cluster.service.ClusterService;
11+
import org.elasticsearch.common.settings.Settings;
12+
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
13+
import org.elasticsearch.threadpool.ThreadPool;
14+
import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
15+
import org.elasticsearch.xpack.core.template.IndexTemplateRegistry;
16+
import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig;
17+
18+
import java.util.List;
19+
20+
import static org.elasticsearch.xpack.core.ClientHelper.DEPRECATION_ORIGIN;
21+
22+
/**
23+
* Manages the index template and associated ILM policy for deprecation log indexing.
24+
*/
25+
public class DeprecationIndexingTemplateRegistry extends IndexTemplateRegistry {
26+
// history (please add a comment why you increased the version here)
27+
// version 1: initial
28+
public static final int INDEX_TEMPLATE_VERSION = 1;
29+
30+
public static final String DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE = "xpack.deprecation.indexing.template.version";
31+
32+
public static final String DEPRECATION_INDEXING_MAPPINGS_NAME = ".deprecation-indexing-mappings";
33+
public static final String DEPRECATION_INDEXING_SETTINGS_NAME = ".deprecation-indexing-settings";
34+
public static final String DEPRECATION_INDEXING_TEMPLATE_NAME = ".deprecation-indexing-template";
35+
public static final String DEPRECATION_INDEXING_POLICY_NAME = ".deprecation-indexing-ilm-policy";
36+
37+
public static final IndexTemplateConfig DEPRECATION_INDEXING_MAPPINGS = new IndexTemplateConfig(
38+
DEPRECATION_INDEXING_MAPPINGS_NAME,
39+
"/org/elasticsearch/xpack/deprecation/deprecation-indexing-mappings.json",
40+
INDEX_TEMPLATE_VERSION,
41+
DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE
42+
);
43+
44+
public static final IndexTemplateConfig DEPRECATION_INDEXING_SETTINGS = new IndexTemplateConfig(
45+
DEPRECATION_INDEXING_SETTINGS_NAME,
46+
"/org/elasticsearch/xpack/deprecation/deprecation-indexing-settings.json",
47+
INDEX_TEMPLATE_VERSION,
48+
DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE
49+
);
50+
51+
public static final IndexTemplateConfig DEPRECATION_INDEXING_INDEX_TEMPLATE = new IndexTemplateConfig(
52+
DEPRECATION_INDEXING_TEMPLATE_NAME,
53+
"/org/elasticsearch/xpack/deprecation/deprecation-indexing-template.json",
54+
INDEX_TEMPLATE_VERSION,
55+
DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE
56+
);
57+
58+
public static final LifecyclePolicyConfig DEPRECATION_INDEXING_HISTORY_POLICY = new LifecyclePolicyConfig(
59+
DEPRECATION_INDEXING_POLICY_NAME,
60+
"/org/elasticsearch/xpack/deprecation/deprecation-indexing-ilm-policy.json"
61+
);
62+
63+
public DeprecationIndexingTemplateRegistry(
64+
Settings nodeSettings,
65+
ClusterService clusterService,
66+
ThreadPool threadPool,
67+
Client client,
68+
NamedXContentRegistry xContentRegistry
69+
) {
70+
super(nodeSettings, clusterService, threadPool, client, xContentRegistry);
71+
}
72+
73+
@Override
74+
protected List<IndexTemplateConfig> getComponentTemplateConfigs() {
75+
return List.of(DEPRECATION_INDEXING_MAPPINGS, DEPRECATION_INDEXING_SETTINGS);
76+
}
77+
78+
@Override
79+
protected List<IndexTemplateConfig> getComposableTemplateConfigs() {
80+
return List.of(DEPRECATION_INDEXING_INDEX_TEMPLATE);
81+
}
82+
83+
@Override
84+
protected List<LifecyclePolicyConfig> getPolicyConfigs() {
85+
return List.of(DEPRECATION_INDEXING_HISTORY_POLICY);
86+
}
87+
88+
@Override
89+
protected String getOrigin() {
90+
return DEPRECATION_ORIGIN;
91+
}
92+
}

0 commit comments

Comments
 (0)