Skip to content

Commit

Permalink
Migrate logstash system index to be auto-created (#66510)
Browse files Browse the repository at this point in the history 
Backport of #66190.

Part of #61656. Auto-create the `.logstash` index using the system index
infrastructure.
  • Loading branch information
@pugnascotia
pugnascotia authored Jan 5, 2021
1 parent f687b93 commit aa3eb56
Show file tree
Hide file tree
Showing 4 changed files with 106 additions and 94 deletions.
2 changes: 0 additions & 2 deletions test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1368,8 +1368,6 @@ protected static boolean isXPackTemplate(String name) {
}
switch (name) {
case ".watches":
case "logstash-index-template":
case ".logstash-management":
case "security_audit_log":
case ".slm-history":
case ".async-search":
Expand Down
51 changes: 0 additions & 51 deletions x-pack/plugin/core/src/main/resources/logstash-management.json
View file

This file was deleted.

138 changes: 106 additions & 32 deletions x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,25 @@
*/
package org.elasticsearch.xpack.logstash;

import org.apache.logging.log4j.LogManager;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsFilter;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.codec.CodecService;
import org.elasticsearch.indices.SystemIndexDescriptor;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.SystemIndexPlugin;
import org.elasticsearch.xpack.core.XPackPlugin;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestHandler;
import org.elasticsearch.xpack.core.template.TemplateUtils;
import org.elasticsearch.xpack.core.XPackPlugin;
import org.elasticsearch.xpack.logstash.action.DeletePipelineAction;
import org.elasticsearch.xpack.logstash.action.GetPipelineAction;
import org.elasticsearch.xpack.logstash.action.PutPipelineAction;
Expand All @@ -35,24 +34,25 @@
import org.elasticsearch.xpack.logstash.rest.RestGetPipelineAction;
import org.elasticsearch.xpack.logstash.rest.RestPutPipelineAction;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import java.util.function.UnaryOperator;

import static java.util.Collections.singletonList;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.engine.EngineConfig.INDEX_CODEC_SETTING;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.elasticsearch.xpack.core.ClientHelper.LOGSTASH_MANAGEMENT_ORIGIN;

/**
* This class activates/deactivates the logstash modules depending if we're running a node client or transport client
*/
public class Logstash extends Plugin implements SystemIndexPlugin {

public static final String LOGSTASH_CONCRETE_INDEX_NAME = ".logstash";
private static final String LOGSTASH_TEMPLATE_FILE_NAME = "logstash-management";
private static final String LOGSTASH_INDEX_TEMPLATE_NAME = ".logstash-management";
private static final String OLD_LOGSTASH_INDEX_NAME = "logstash-index-template";
private static final String TEMPLATE_VERSION_VARIABLE = "logstash.template.version";

public Logstash() {}

Expand Down Expand Up @@ -88,27 +88,101 @@ public List<RestHandler> getRestHandlers(
);
}

public UnaryOperator<Map<String, IndexTemplateMetadata>> getIndexTemplateMetadataUpgrader() {
return templates -> {
templates.keySet().removeIf(OLD_LOGSTASH_INDEX_NAME::equals);
TemplateUtils.loadTemplateIntoMap(
"/" + LOGSTASH_TEMPLATE_FILE_NAME + ".json",
templates,
LOGSTASH_INDEX_TEMPLATE_NAME,
Version.CURRENT.toString(),
TEMPLATE_VERSION_VARIABLE,
LogManager.getLogger(Logstash.class)
);
// internal representation of typeless templates requires the default "_doc" type, which is also required for internal templates
assert templates.get(LOGSTASH_INDEX_TEMPLATE_NAME).mappings().get(MapperService.SINGLE_MAPPING_NAME) != null;
return templates;
};
}

@Override
public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
return Collections.singletonList(
new SystemIndexDescriptor(LOGSTASH_CONCRETE_INDEX_NAME, "Contains data for Logstash Central Management")
return singletonList(
SystemIndexDescriptor.builder()
.setIndexPattern(LOGSTASH_CONCRETE_INDEX_NAME)
.setPrimaryIndex(LOGSTASH_CONCRETE_INDEX_NAME)
.setDescription("Contains data for Logstash Central Management")
.setMappings(getIndexMappings())
.setSettings(getIndexSettings())
.setVersionMetaKey("logstash-version")
.setOrigin(LOGSTASH_MANAGEMENT_ORIGIN)
.build()
);
}

private Settings getIndexSettings() {
return Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
.put(INDEX_CODEC_SETTING.getKey(), CodecService.BEST_COMPRESSION_CODEC)
.build();
}

private XContentBuilder getIndexMappings() {
try {
final XContentBuilder builder = jsonBuilder();
{
builder.startObject();
{
builder.startObject(SINGLE_MAPPING_NAME);
builder.field("dynamic", "strict");
{
builder.startObject("_meta");
builder.field("logstash-version", Version.CURRENT);
builder.endObject();
}
{
builder.startObject("properties");
{
builder.startObject("description");
builder.field("type", "text");
builder.endObject();
}
{
builder.startObject("last_modified");
builder.field("type", "date");
builder.endObject();
}
{
builder.startObject("pipeline_metadata");
{
builder.startObject("properties");
{
builder.startObject("version");
builder.field("type", "short");
builder.endObject();
builder.startObject("type");
builder.field("type", "keyword");
builder.endObject();
}
builder.endObject();
}
builder.endObject();
}
{
builder.startObject("pipeline");
builder.field("type", "text");
builder.endObject();
}
{
builder.startObject("pipeline_settings");
builder.field("dynamic", false);
builder.field("type", "object");
builder.endObject();
}
{
builder.startObject("username");
builder.field("type", "keyword");
builder.endObject();
}
{
builder.startObject("metadata");
builder.field("dynamic", false);
builder.field("type", "object");
builder.endObject();
}
builder.endObject();
}
builder.endObject();
}
builder.endObject();
}
return builder;
} catch (IOException e) {
throw new UncheckedIOException("Failed to build " + LOGSTASH_CONCRETE_INDEX_NAME + " index mappings", e);
}
}
}
9 changes: 0 additions & 9 deletions ...plugin/src/javaRestTest/java/org/elasticsearch/xpack/test/rest/LogstashSystemIndexIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,6 @@ protected Settings restClientSettings() {
.build();
}

public void testTemplateIsPut() throws Exception {
assertBusy(
() -> assertThat(
client().performRequest(new Request("HEAD", "/_template/.logstash-management")).getStatusLine().getStatusCode(),
is(200)
)
);
}

public void testPipelineCRUD() throws Exception {
// put pipeline
final String pipelineJson = getPipelineJson();
Expand Down

0 comments on commit aa3eb56

Please sign in to comment.