Logs/indexer appender

distribution/src/config/log4j2.properties

@@ -72,10 +72,21 @@ appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy
 appender.deprecation_rolling.strategy.max = 4
 #################################################
 
+
 logger.deprecation.name = org.elasticsearch.deprecation
 logger.deprecation.level = warn
 logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
 logger.deprecation.additivity = false
+############
+appender.deprecation_indexer.type = DeprecationIndexer
+appender.deprecation_indexer.name = deprecation_indexer
+appender.deprecation_indexer.layout.type = ECSJsonLayout
+appender.deprecation_indexer.layout.type_name = deprecation_indexer
+
+logger.deprecation_indexer.name = deprecation_indexer
+logger.deprecation_indexer.level = warn
+logger.deprecation_indexer.appenderRef.deprecation_indexer.ref = deprecation_indexer
+logger.deprecation_indexer.additivity = false
 
 ######## Search slowlog JSON ####################
 appender.index_search_slowlog_rolling.type = RollingFile

server/src/main/java/org/elasticsearch/common/logging/DeprecationAppender.java

@@ -0,0 +1,115 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.logging;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.Core;
+import org.apache.logging.log4j.core.Filter;
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.appender.AbstractAppender;
+import org.apache.logging.log4j.core.appender.FileAppender;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.index.IndexAction;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.xcontent.XContentType;
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.common.Strings.isNullOrEmpty;
+
+@Plugin(name = DeprecationAppender.NAME, category = Core.CATEGORY_NAME, elementType = Appender.ELEMENT_TYPE, printObject = true)
+public class DeprecationAppender extends AbstractAppender{
+    public static final String NAME = "DeprecationIndexer";
+    private static final Logger logger = LogManager.getLogger(DeprecationAppender.class);
+
+    private static final String TEMPLATE_NAME = ".deprecation_logs";
+
+//    private final ClusterService clusterService;
+//    private final NodeClient nodeClient;
+
+    private boolean isTemplateCreated = false;
+    private NodeClient nodeClient;
+
+    public DeprecationAppender(String name, Filter filter, Layout<? extends Serializable> layout) {
+        super(name, filter, layout);
+    }
+
+    @Override
+    public void start() {
+        this.setStarting();
+        if (getFilter() != null) {
+            getFilter().start();
+        }
+//        this.setStarted();
+    }
+
+    public void start(NodeClient client) {
+        this.nodeClient = client;
+        setStarted();//sets volatile variable
+    }
+
+    public static class Builder<B extends DeprecationAppender.Builder<B>> extends AbstractAppender.Builder<B>
+        implements org.apache.logging.log4j.core.util.Builder<DeprecationAppender> {
+
+        @Override
+        public DeprecationAppender build() {
+            return new DeprecationAppender(getName(),getFilter(),getOrCreateLayout());
+        }
+    }
+
+    @PluginBuilderFactory
+    public static <B extends DeprecationAppender.Builder<B>> B newBuilder() {
+        return new DeprecationAppender.Builder<B>().asBuilder();
+    }
+
+    @Override
+    public void append(LogEvent event) {
+        byte[] payload = getLayout().toByteArray(event);
+        final String indexName = TEMPLATE_NAME + "." + DateTimeFormatter.ISO_LOCAL_DATE.format(LocalDate.now());
+
+        new IndexRequestBuilder(nodeClient, IndexAction.INSTANCE).setIndex(indexName)
+            .setOpType(DocWriteRequest.OpType.CREATE)
+            .setSource(payload, XContentType.JSON)
+            .execute(new ActionListener<>() {
+                @Override
+                public void onResponse(IndexResponse indexResponse) {
+                    // Nothing to do
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    logger.error("Failed to index deprecation message", e);
+                }
+            });
+    }
+}

server/src/main/java/org/elasticsearch/common/logging/DeprecationIndexerAppender.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.logging;
+
+import org.apache.logging.log4j.core.Filter;
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.appender.AbstractAppender;
+
+import java.io.Serializable;
+
+public class DeprecationIndexerAppender extends AbstractAppender {
+
+    protected DeprecationIndexerAppender(String name, Filter filter, Layout<? extends Serializable> layout) {
+        super(name, filter, layout);
+    }
+
+    @Override
+    public void append(LogEvent event) {
+
+    }
+}

server/src/main/java/org/elasticsearch/common/logging/DeprecationLayout.java

@@ -0,0 +1,4 @@
+package org.elasticsearch.common.logging;
+
+public class DeprecationLayout {
+}

server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java

@@ -37,6 +37,7 @@
  * @see ThrottlingAndHeaderWarningLogger for throttling and header warnings implementation details
  */
 public class DeprecationLogger {
+    private final Logger deprecationIndexer = LogManager.getLogger("deprecation_indexer");
     private final ThrottlingAndHeaderWarningLogger deprecationLogger;
 
     /**
@@ -47,6 +48,7 @@ public class DeprecationLogger {
      */
     public DeprecationLogger(Logger parentLogger) {
         deprecationLogger = new ThrottlingAndHeaderWarningLogger(deprecatedLoggerName(parentLogger));
+
     }
 
     private static Logger deprecatedLoggerName(Logger parentLogger) {
@@ -83,6 +85,7 @@ public DeprecationLoggerBuilder withDeprecation(String key, String msg, Object[]
             String opaqueId = HeaderWarning.getXOpaqueId();
             ESLogMessage deprecationMessage = DeprecatedMessage.of(opaqueId, msg, params);
             deprecationLogger.throttleLogAndAddWarning(key, deprecationMessage);
+            deprecationIndexer.warn(deprecationMessage);
             return this;
         }
 

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -59,6 +59,7 @@
 import org.elasticsearch.cluster.service.ClusterApplierService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterService;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.network.NetworkService;
@@ -196,6 +197,7 @@ public void apply(Settings value, Settings current, Settings previous) {
             ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING,
             ConcurrentRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING,
             DanglingIndicesState.AUTO_IMPORT_DANGLING_INDICES_SETTING,
+//            DeprecationLogger.WRITE_DEPRECATION_LOGS_TO_INDEX,
             EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING,
             EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING,
             FilterAllocationDecider.CLUSTER_ROUTING_INCLUDE_GROUP_SETTING,

server/src/main/java/org/elasticsearch/node/Node.java

@@ -21,6 +21,8 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.Assertions;
@@ -69,6 +71,7 @@
 import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.logging.DeprecationAppender;
 import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.common.logging.NodeAndClusterIdStateListener;
 import org.elasticsearch.common.network.NetworkAddress;
@@ -652,6 +655,18 @@ protected Node(final Environment initialEnvironment,
             actionModule.initRestHandlers(() -> clusterService.state().nodes());
             logger.info("initialized");
 
+            final LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+            final Configuration config = ctx.getConfiguration();
+            DeprecationAppender deprecation_appender = config.getAppender("deprecation_indexer");
+            deprecation_appender.start(client);
+
+            resourcesToClose.add(() -> deprecation_appender.stop());
+
+//            Logger rootLogger = LogManager.getRootLogger();
+//            final DeprecationAppender deprecationAppender = new DeprecationAppender(clusterService, client);
+//            DeprecationLogger.setIndexer(deprecationAppender);
+//            resourcesToClose.add(() -> DeprecationLogger.removeIndexer(deprecationAppender));
+
             success = true;
         } catch (IOException ex) {
             throw new ElasticsearchException("failed to bind service", ex);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java

@@ -21,6 +21,8 @@ public final class SystemPrivilege extends Privilege {
 
     private static final Predicate<String> ALLOWED_ACTIONS = Automatons.predicate(
         "internal:*",
+        "indices:*", // DIRTY GREAT HACK
+
         "indices:monitor/*", // added for monitoring
         "cluster:monitor/*",  // added for monitoring
         "cluster:admin/bootstrap/*", // for the bootstrap service