From 006da9b067bc703f0e4a71e356c3e87b6f145a01 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Mon, 12 Aug 2019 15:08:52 +0300 Subject: [PATCH 1/3] Main --- distribution/build.gradle | 1 + .../core/src/main/config/log4j2.properties | 7 +++ .../audit/logfile/LoggingAuditTrail.java | 46 ++++++++++--------- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/distribution/build.gradle b/distribution/build.gradle index e6ac88228c96a..50e54be549ab3 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -425,6 +425,7 @@ task run(type: RunTask) { setupCommand 'setupTestAdmin', 'bin/elasticsearch-users', 'useradd', 'elastic-admin', '-p', 'elastic-password', '-r', 'superuser' setting 'xpack.security.enabled', 'true' + setting 'xpack.security.audit.enabled', 'true' setting 'xpack.monitoring.enabled', 'true' setting 'xpack.sql.enabled', 'true' setting 'xpack.rollup.enabled', 'true' diff --git a/x-pack/plugin/core/src/main/config/log4j2.properties b/x-pack/plugin/core/src/main/config/log4j2.properties index c37faf84afbea..f826694e5e1e7 100644 --- a/x-pack/plugin/core/src/main/config/log4j2.properties +++ b/x-pack/plugin/core/src/main/config/log4j2.properties @@ -76,6 +76,13 @@ logger.xpack_security_audit_logfile.level = info logger.xpack_security_audit_logfile.appenderRef.audit_rolling.ref = audit_rolling logger.xpack_security_audit_logfile.additivity = false +filters = xpack_only_security_audit_filter +filter.xpack_only_security_audit_filter.type = MarkerFilter +filter.xpack_only_security_audit_filter.name = xpack_only_security_audit_filter +filter.xpack_only_security_audit_filter.marker = AUDIT +filter.xpack_only_security_audit_filter.onMatch = ACCEPT +filter.xpack_only_security_audit_filter.onMismatch = NEUTRAL + logger.xmlsig.name = org.apache.xml.security.signature.XMLSignature logger.xmlsig.level = error logger.samlxml_decrypt.name = org.opensaml.xmlsec.encryption.support.Decrypter diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index 11f128e572077..6bce59eae4207 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -8,6 +8,8 @@ import com.fasterxml.jackson.core.io.JsonStringEncoder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.MarkerManager; import org.apache.logging.log4j.message.StringMapMessage; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.cluster.ClusterChangedEvent; @@ -151,6 +153,8 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener { "indices", (key) -> Setting.listSetting(key, Collections.singletonList("*"), Function.identity(), Property.NodeScope, Property.Dynamic)); + private static final Marker AUDIT_MARKER = MarkerManager.getMarker("AUDIT"); + private final Logger logger; private final ThreadContext threadContext; final EventFilterPolicyRegistry eventFilterPolicyRegistry; @@ -225,7 +229,7 @@ public void authenticationSuccess(String requestId, String realm, User user, Res .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } @@ -248,7 +252,7 @@ public void authenticationSuccess(String requestId, String realm, User user, Str .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } } @@ -270,7 +274,7 @@ public void anonymousAccessDenied(String requestId, String action, TransportMess .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } } @@ -289,7 +293,7 @@ public void anonymousAccessDenied(String requestId, RestRequest request) { .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } @@ -311,7 +315,7 @@ public void authenticationFailed(String requestId, AuthenticationToken token, St .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } } @@ -329,7 +333,7 @@ public void authenticationFailed(String requestId, RestRequest request) { .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } @@ -350,7 +354,7 @@ public void authenticationFailed(String requestId, String action, TransportMessa .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } } @@ -370,7 +374,7 @@ public void authenticationFailed(String requestId, AuthenticationToken token, Re .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } @@ -393,7 +397,7 @@ public void authenticationFailed(String requestId, String realm, AuthenticationT .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } } @@ -414,7 +418,7 @@ public void authenticationFailed(String requestId, String realm, AuthenticationT .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } @@ -440,7 +444,7 @@ public void accessGranted(String requestId, Authentication authentication, Strin .withXForwardedFor(threadContext) .with(authorizationInfo.asMap()) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } } @@ -480,7 +484,7 @@ public void explicitIndexAccessEvent(String requestId, AuditLevel eventType, Aut .with(ORIGIN_TYPE_FIELD_NAME, TRANSPORT_ORIGIN_FIELD_VALUE) .with(ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(remoteAddress.address())); } - logger.info(logEntryBuilder.build()); + logger.info(AUDIT_MARKER, logEntryBuilder.build()); } } } @@ -505,7 +509,7 @@ public void accessDenied(String requestId, Authentication authentication, String .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } } @@ -523,7 +527,7 @@ public void tamperedRequest(String requestId, RestRequest request) { .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } @@ -544,7 +548,7 @@ public void tamperedRequest(String requestId, String action, TransportMessage me .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } } @@ -567,7 +571,7 @@ public void tamperedRequest(String requestId, User user, String action, Transpor .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } } @@ -586,7 +590,7 @@ public void connectionGranted(InetAddress inetAddress, String profile, SecurityI .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } @@ -604,7 +608,7 @@ public void connectionDenied(InetAddress inetAddress, String profile, SecurityIp .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } @@ -628,7 +632,7 @@ public void runAsGranted(String requestId, Authentication authentication, String .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } } @@ -653,7 +657,7 @@ public void runAsDenied(String requestId, Authentication authentication, String .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } } @@ -675,7 +679,7 @@ public void runAsDenied(String requestId, Authentication authentication, RestReq .withOpaqueId(threadContext) .withXForwardedFor(threadContext) .build(); - logger.info(logEntry); + logger.info(AUDIT_MARKER, logEntry); } } From 692ae73f3945fba5139af36a11bb1a99c87618fd Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Mon, 12 Aug 2019 17:18:21 +0300 Subject: [PATCH 2/3] Code only, no config --- distribution/build.gradle | 1 - .../plugin/core/src/main/config/log4j2.properties | 7 ------- .../security/audit/logfile/LoggingAuditTrail.java | 15 ++++++++++++++- .../audit/logfile/LoggingAuditTrailTests.java | 2 +- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/distribution/build.gradle b/distribution/build.gradle index 50e54be549ab3..e6ac88228c96a 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -425,7 +425,6 @@ task run(type: RunTask) { setupCommand 'setupTestAdmin', 'bin/elasticsearch-users', 'useradd', 'elastic-admin', '-p', 'elastic-password', '-r', 'superuser' setting 'xpack.security.enabled', 'true' - setting 'xpack.security.audit.enabled', 'true' setting 'xpack.monitoring.enabled', 'true' setting 'xpack.sql.enabled', 'true' setting 'xpack.rollup.enabled', 'true' diff --git a/x-pack/plugin/core/src/main/config/log4j2.properties b/x-pack/plugin/core/src/main/config/log4j2.properties index f826694e5e1e7..c37faf84afbea 100644 --- a/x-pack/plugin/core/src/main/config/log4j2.properties +++ b/x-pack/plugin/core/src/main/config/log4j2.properties @@ -76,13 +76,6 @@ logger.xpack_security_audit_logfile.level = info logger.xpack_security_audit_logfile.appenderRef.audit_rolling.ref = audit_rolling logger.xpack_security_audit_logfile.additivity = false -filters = xpack_only_security_audit_filter -filter.xpack_only_security_audit_filter.type = MarkerFilter -filter.xpack_only_security_audit_filter.name = xpack_only_security_audit_filter -filter.xpack_only_security_audit_filter.marker = AUDIT -filter.xpack_only_security_audit_filter.onMatch = ACCEPT -filter.xpack_only_security_audit_filter.onMismatch = NEUTRAL - logger.xmlsig.name = org.apache.xml.security.signature.XMLSignature logger.xmlsig.level = error logger.samlxml_decrypt.name = org.opensaml.xmlsec.encryption.support.Decrypter diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index 6bce59eae4207..87fb8b7572357 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -10,6 +10,9 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Marker; import org.apache.logging.log4j.MarkerManager; +import org.apache.logging.log4j.core.Filter.Result; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.filter.MarkerFilter; import org.apache.logging.log4j.message.StringMapMessage; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.cluster.ClusterChangedEvent; @@ -18,6 +21,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -37,6 +41,7 @@ import org.elasticsearch.xpack.core.security.user.SystemUser; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.security.user.XPackUser; +import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.audit.AuditLevel; import org.elasticsearch.xpack.security.audit.AuditTrail; import org.elasticsearch.xpack.security.rest.RemoteHostHeader; @@ -170,7 +175,7 @@ public String name() { } public LoggingAuditTrail(Settings settings, ClusterService clusterService, ThreadPool threadPool) { - this(settings, clusterService, LogManager.getLogger(), threadPool.getThreadContext()); + this(settings, clusterService, LogManager.getLogger(LoggingAuditTrail.class), threadPool.getThreadContext()); } LoggingAuditTrail(Settings settings, ClusterService clusterService, Logger logger, ThreadContext threadContext) { @@ -211,6 +216,14 @@ public LoggingAuditTrail(Settings settings, ClusterService clusterService, Threa final EventFilterPolicy newPolicy = policy.orElse(new EventFilterPolicy(policyName, settings)).changeIndicesFilter(filtersList); this.eventFilterPolicyRegistry.set(policyName, newPolicy); }, (policyName, filtersList) -> EventFilterPolicy.parsePredicate(filtersList)); + // this log filter ensures that audit events are not filtered out because of the log level + final LoggerContext ctx = LoggerContext.getContext(false); + MarkerFilter auditMarkerFilter = MarkerFilter.createFilter(AUDIT_MARKER.getName(), Result.ACCEPT, Result.NEUTRAL); + ctx.addFilter(auditMarkerFilter); + ctx.updateLoggers(); + clusterService.getClusterSettings().addSettingsUpdateConsumer(ignored -> { + LogManager.getLogger(Security.class).warn("Changing log level for [" + LoggingAuditTrail.class.getName() + "] has no effect"); + }, List.of(Loggers.LOG_LEVEL_SETTING.getConcreteSettingForNamespace(LoggingAuditTrail.class.getName()))); } @Override diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java index e57ea456fe97e..230cb4c4950d2 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java @@ -199,7 +199,7 @@ public void init() throws Exception { threadContext.putHeader(AuditTrail.X_FORWARDED_FOR_HEADER, randomFrom("2001:db8:85a3:8d3:1319:8a2e:370:7348", "203.0.113.195", "203.0.113.195, 70.41.3.18, 150.172.238.178")); } - logger = CapturingLogger.newCapturingLogger(Level.INFO, patternLayout); + logger = CapturingLogger.newCapturingLogger(randomFrom(Level.OFF, Level.FATAL, Level.ERROR, Level.WARN, Level.INFO), patternLayout); auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); } From 0b63ae6def00f07daaef2329867a5c17af7920dc Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Sun, 20 Oct 2019 15:23:49 +0300 Subject: [PATCH 3/3] Rename marker --- .../xpack/security/audit/logfile/LoggingAuditTrail.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index 87fb8b7572357..b171f58a4ade7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -158,7 +158,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener { "indices", (key) -> Setting.listSetting(key, Collections.singletonList("*"), Function.identity(), Property.NodeScope, Property.Dynamic)); - private static final Marker AUDIT_MARKER = MarkerManager.getMarker("AUDIT"); + private static final Marker AUDIT_MARKER = MarkerManager.getMarker("org.elasticsearch.xpack.security.audit"); private final Logger logger; private final ThreadContext threadContext;