diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index b673900dc100d..8395ea87e63fb 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2478,11 +2478,6 @@ - - - - - diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java index c0b52c80d89a9..ac033d5091a47 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java @@ -144,7 +144,7 @@ public void testLoggingLevelsFromSettings() throws IOException, UserException { final LoggerContext ctx = (LoggerContext) LogManager.getContext(false); final Configuration config = ctx.getConfiguration(); final Map loggerConfigs = config.getLoggers(); - assertThat(loggerConfigs.size(), equalTo(5)); + assertThat(loggerConfigs.size(), equalTo(3)); assertThat(loggerConfigs, hasKey("")); assertThat(loggerConfigs.get("").getLevel(), equalTo(rootLevel)); assertThat(loggerConfigs, hasKey("foo")); diff --git a/server/build.gradle b/server/build.gradle index 68ed895516fba..3b1885a2fc7ea 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -64,7 +64,6 @@ dependencies { // logging api "org.apache.logging.log4j:log4j-api:${versions.log4j}" api "org.apache.logging.log4j:log4j-core:${versions.log4j}" - implementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}" api "net.java.dev.jna:jna:${versions.jna}" @@ -249,7 +248,6 @@ tasks.named("thirdPartyAudit").configure { tasks.named("dependencyLicenses").configure { mapping from: /lucene-.*/, to: 'lucene' - mapping from: /log4j-.*/, to: 'log4j' dependencies = project.configurations.runtimeClasspath.fileCollection { it.group.startsWith('org.elasticsearch') == false || // keep the following org.elasticsearch jars in diff --git a/server/licenses/log4j-LICENSE.txt b/server/licenses/log4j-api-LICENSE.txt similarity index 100% rename from server/licenses/log4j-LICENSE.txt rename to server/licenses/log4j-api-LICENSE.txt diff --git a/server/licenses/log4j-NOTICE.txt b/server/licenses/log4j-api-NOTICE.txt similarity index 100% rename from server/licenses/log4j-NOTICE.txt rename to server/licenses/log4j-api-NOTICE.txt diff --git a/server/licenses/log4j-core-LICENSE.txt b/server/licenses/log4j-core-LICENSE.txt new file mode 100644 index 0000000000000..6279e5206de13 --- /dev/null +++ b/server/licenses/log4j-core-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 1999-2005 The Apache Software Foundation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/server/licenses/log4j-core-NOTICE.txt b/server/licenses/log4j-core-NOTICE.txt new file mode 100644 index 0000000000000..bbb5fb3f66e2a --- /dev/null +++ b/server/licenses/log4j-core-NOTICE.txt @@ -0,0 +1,20 @@ +Apache Log4j +Copyright 1999-2023 Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +ResolverUtil.java +Copyright 2005-2006 Tim Fennell + +Dumbster SMTP test server +Copyright 2004 Jason Paul Kitchen + +TypeUtil.java +Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams + +picocli (http://picocli.info) +Copyright 2017 Remko Popma + +TimeoutBlockingWaitStrategy.java and parts of Util.java +Copyright 2011 LMAX Ltd. diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index b2b1419b620a8..42125556f771c 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -39,7 +39,6 @@ requires org.apache.logging.log4j; requires org.apache.logging.log4j.core; - requires org.apache.logging.log4j.jul; requires org.apache.lucene.analysis.common; requires org.apache.lucene.backward_codecs; diff --git a/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java b/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java index 6b7ff4ad35dab..a79c6d1d16940 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java +++ b/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java @@ -12,7 +12,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.Appender; -import org.apache.logging.log4j.core.Filter; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.appender.ConsoleAppender; import org.apache.logging.log4j.core.config.AbstractConfiguration; @@ -22,13 +21,11 @@ import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder; import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory; import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration; -import org.apache.logging.log4j.core.config.builder.impl.DefaultConfigurationBuilder; import org.apache.logging.log4j.core.config.composite.CompositeConfiguration; import org.apache.logging.log4j.core.config.plugins.util.PluginManager; import org.apache.logging.log4j.core.config.properties.PropertiesConfiguration; import org.apache.logging.log4j.core.config.properties.PropertiesConfigurationBuilder; import org.apache.logging.log4j.core.config.properties.PropertiesConfigurationFactory; -import org.apache.logging.log4j.jul.Log4jBridgeHandler; import org.apache.logging.log4j.status.StatusConsoleListener; import org.apache.logging.log4j.status.StatusData; import org.apache.logging.log4j.status.StatusListener; @@ -225,7 +222,6 @@ public PropertiesConfiguration getConfiguration(final LoggerContext loggerContex properties.setProperty(name, value.replace("%marker", "[%node_name]%marker ")); } } - // end hack return new PropertiesConfigurationBuilder().setConfigurationSource(source) .setRootProperties(properties) @@ -245,8 +241,6 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr }); assert configurations.isEmpty() == false; - configurations.add(createStaticConfiguration(context)); - context.start(new CompositeConfiguration(configurations)); configureLoggerLevels(settings); @@ -263,13 +257,26 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr ); } - Log4jBridgeHandler.install(true, "", true); - // Redirect stdout/stderr to log4j. While we ensure Elasticsearch code does not write to those streams, // third party libraries may do that. Note that we do NOT close the streams because other code may have // grabbed a handle to the streams and intend to write to it, eg log4j for writing to the console - System.setOut(new PrintStream(new LoggingOutputStream(LogManager.getLogger("stdout"), Level.INFO), false, StandardCharsets.UTF_8)); - System.setErr(new PrintStream(new LoggingOutputStream(LogManager.getLogger("stderr"), Level.WARN), false, StandardCharsets.UTF_8)); + System.setOut( + new PrintStream(new LoggingOutputStream(LogManager.getLogger("stdout"), Level.INFO, List.of()), false, StandardCharsets.UTF_8) + ); + System.setErr( + new PrintStream( + new LoggingOutputStream( + LogManager.getLogger("stderr"), + Level.WARN, + // MMapDirectory messages come from Lucene, suggesting to users as a warning that they should enable preview features in + // the JDK. Vector logs come from Lucene too, but only if the used explicitly disables the Vector API - no point warning + // in this case. + List.of("MMapDirectory", "VectorUtilProvider", "WARNING: Java vector incubator module is not readable") + ), + false, + StandardCharsets.UTF_8 + ) + ); final Logger rootLogger = LogManager.getRootLogger(); Appender appender = Loggers.findAppender(rootLogger, ConsoleAppender.class); @@ -282,29 +289,6 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr } } - /** - * Creates a log4j configuration that is not changeable by users. - */ - private static AbstractConfiguration createStaticConfiguration(LoggerContext context) { - var builder = new DefaultConfigurationBuilder<>(); - builder.setConfigurationSource(ConfigurationSource.NULL_SOURCE); - builder.setLoggerContext(context); - - // adding filters for confusing Lucene messages - addRegexFilter(builder, "org.apache.lucene.store.MemorySegmentIndexInputProvider", "Using MemorySegmentIndexInput.*"); - addRegexFilter(builder, "org.apache.lucene.util.VectorUtilProvider", ".* incubator module is not readable.*"); - - return builder.build(); - } - - private static void addRegexFilter(DefaultConfigurationBuilder builder, String loggerName, String pattern) { - var filterBuilder = builder.newFilter("RegexFilter", Filter.Result.DENY, Filter.Result.NEUTRAL); - filterBuilder.addAttribute("regex", pattern); - var loggerBuilder = builder.newLogger(loggerName); - loggerBuilder.add(filterBuilder); - builder.add(loggerBuilder); - } - /** * Removes the appender for the console, if one exists. */ diff --git a/server/src/main/java/org/elasticsearch/common/logging/LoggingOutputStream.java b/server/src/main/java/org/elasticsearch/common/logging/LoggingOutputStream.java index 4092d1d30192d..6fd98259d9250 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/LoggingOutputStream.java +++ b/server/src/main/java/org/elasticsearch/common/logging/LoggingOutputStream.java @@ -15,6 +15,7 @@ import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.List; /** * A stream whose output is sent to the configured logger, line by line. @@ -42,9 +43,12 @@ static class Buffer { private final Level level; - LoggingOutputStream(Logger logger, Level level) { + private final List messageFilters; + + LoggingOutputStream(Logger logger, Level level, List messageFilters) { this.logger = logger; this.level = level; + this.messageFilters = messageFilters; } @Override @@ -103,8 +107,17 @@ public void close() { threadLocal = null; } + private void log(String msg) { + for (String filter : messageFilters) { + if (msg.contains(filter)) { + return; + } + } + this.log0(msg); + } + // pkg private for testing - protected void log(String msg) { + protected void log0(String msg) { logger.log(level, msg); } } diff --git a/server/src/test/java/org/elasticsearch/common/logging/LoggingOutputStreamTests.java b/server/src/test/java/org/elasticsearch/common/logging/LoggingOutputStreamTests.java index 233b1b69cb767..62e3046dc63fd 100644 --- a/server/src/test/java/org/elasticsearch/common/logging/LoggingOutputStreamTests.java +++ b/server/src/test/java/org/elasticsearch/common/logging/LoggingOutputStreamTests.java @@ -29,20 +29,22 @@ class TestLoggingOutputStream extends LoggingOutputStream { List lines = new ArrayList<>(); TestLoggingOutputStream() { - super(null, null); + super(null, null, messageFilters); } @Override - protected void log(String msg) { + protected void log0(String msg) { lines.add(msg); } } + List messageFilters = new ArrayList<>(); TestLoggingOutputStream loggingStream; PrintStream printStream; @Before public void createStream() { + messageFilters.clear(); loggingStream = new TestLoggingOutputStream(); printStream = new PrintStream(loggingStream, false, StandardCharsets.UTF_8); } @@ -115,4 +117,12 @@ public void testThreadIsolation() throws Exception { printStream.flush(); assertThat(loggingStream.lines, contains("from thread 2", "from thread 1")); } + + public void testMessageFilters() throws Exception { + messageFilters.add("foo bar"); + printStream.println("prefix foo bar suffix"); + printStream.println("non-filtered message"); + printStream.flush(); + assertThat(loggingStream.lines, contains("non-filtered message")); + } }