accesslog - porting accesslogs capabilities from DT's gateway

docker-compose.yml

@@ -41,7 +41,7 @@ services:
     volumes:
       - datadir:/etc/georchestra
     environment:
-      - JAVA_TOOL_OPTIONS=-Dgeorchestra.datadir=/etc/georchestra -Dspring.profiles.active=docker -Xmx512M -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=0.0.0.0:5005
+      - JAVA_TOOL_OPTIONS=-Dgeorchestra.datadir=/etc/georchestra -Dspring.profiles.active=docker -Xmx512M -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=0.0.0.0:5005 -Dreactor.netty.http.server.accessLogEnabled=true
     restart: always
     ports:
       - 8080:8080
@@ -88,9 +88,8 @@ services:
     restart: always
     ports:
       - 10007:8080
-    
+
   echo:
     image: ealen/echo-server:latest
     ports:
       - 10009:80
-

gateway/pom.xml

@@ -32,6 +32,23 @@
     <dependency>
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-starter-validation</artifactId>
+      <exclusions>
+        <exclusion>
+          <artifactId>spring-boot-starter-logging</artifactId>
+          <!-- exclude transitive dep on logging to remove logback,
+			         we're using log4j2 below to use its json output -->
+          <groupId>org.springframework.boot</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>io.micrometer</groupId>
+      <artifactId>context-propagation</artifactId>
+      <version>1.0.4</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-reactive-httpclient</artifactId>
     </dependency>
     <dependency>
       <groupId>org.springframework.boot</groupId>
@@ -76,7 +93,7 @@
     </dependency>
     <dependency>
       <!-- Annotation processor that generates metadata about classes annotated with @ConfigurationProperties. -->
-      <!-- This metadata is used by IDEs to provide auto-completion and documentation for the properties when editing application.properties 
+      <!-- This metadata is used by IDEs to provide auto-completion and documentation for the properties when editing application.properties
         and application.yaml -->
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-configuration-processor</artifactId>
@@ -95,6 +112,20 @@
       <scope>runtime</scope>
       <optional>true</optional>
     </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-log4j2</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-layout-template-json</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <!-- support for SpringProfile in log4j2-spring.xml -->
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-spring-boot</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-starter-test</artifactId>

gateway/src/main/java/org/georchestra/gateway/autoconfigure/app/FiltersAutoConfiguration.java

@@ -18,8 +18,11 @@
  */
 package org.georchestra.gateway.autoconfigure.app;
 
+import org.georchestra.gateway.filter.global.AccessLogFilter;
+import org.georchestra.gateway.filter.global.AccessLogFilterConfig;
 import org.georchestra.gateway.filter.global.ApplicationErrorGatewayFilterFactory;
 import org.georchestra.gateway.filter.global.LoginParamRedirectGatewayFilterFactory;
+import org.georchestra.gateway.filter.global.RequestIdGlobalFilter;
 import org.georchestra.gateway.filter.global.ResolveTargetGlobalFilter;
 import org.georchestra.gateway.filter.headers.HeaderFiltersConfiguration;
 import org.georchestra.gateway.model.GatewayConfigProperties;
@@ -56,7 +59,7 @@
 @AutoConfiguration
 @AutoConfigureBefore(GatewayAutoConfiguration.class)
 @Import(HeaderFiltersConfiguration.class)
-@EnableConfigurationProperties(GatewayConfigProperties.class)
+@EnableConfigurationProperties({ GatewayConfigProperties.class, AccessLogFilterConfig.class })
 public class FiltersAutoConfiguration {
 
     /**
@@ -129,4 +132,14 @@ StripBasePathGatewayFilterFactory stripBasePathGatewayFilterFactory() {
     ApplicationErrorGatewayFilterFactory applicationErrorGatewayFilterFactory() {
         return new ApplicationErrorGatewayFilterFactory();
     }
+
+    @Bean
+    RequestIdGlobalFilter requestIdGlobalFilter() {
+        return new RequestIdGlobalFilter();
+    }
+
+    @Bean
+    AccessLogFilter accessLogGlobalFilter(AccessLogFilterConfig config) {
+        return new AccessLogFilter(config);
+    }
 }

gateway/src/main/java/org/georchestra/gateway/filter/global/AccessLogFilter.java

@@ -0,0 +1,126 @@
+/*
+ * Copyright (C) 2022 by the geOrchestra PSC
+ *
+ * This file is part of geOrchestra.
+ *
+ * geOrchestra is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option)
+ * any later version.
+ *
+ * geOrchestra is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * geOrchestra.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package org.georchestra.gateway.filter.global;
+
+import static org.springframework.cloud.gateway.support.ServerWebExchangeUtils.GATEWAY_REQUEST_URL_ATTR;
+import static org.springframework.cloud.gateway.support.ServerWebExchangeUtils.GATEWAY_ROUTE_ATTR;
+
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.security.Principal;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import org.georchestra.gateway.model.GeorchestraUsers;
+import org.georchestra.security.model.GeorchestraUser;
+import org.slf4j.MDC;
+import org.springframework.cloud.gateway.filter.GatewayFilterChain;
+import org.springframework.cloud.gateway.filter.GlobalFilter;
+import org.springframework.cloud.gateway.route.Route;
+import org.springframework.http.server.reactive.ServerHttpRequest;
+import org.springframework.http.server.reactive.ServerHttpResponse;
+import org.springframework.security.authentication.AnonymousAuthenticationToken;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.GrantedAuthority;
+import org.springframework.security.core.authority.SimpleGrantedAuthority;
+import org.springframework.web.server.ServerWebExchange;
+
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import reactor.core.publisher.Mono;
+
+@RequiredArgsConstructor
+@Slf4j(topic = "org.georchestra.gateway.accesslog")
+public class AccessLogFilter implements GlobalFilter {
+
+    private final @NonNull AccessLogFilterConfig config;
+
+    @Override
+    public Mono<Void> filter(ServerWebExchange exchange, GatewayFilterChain chain) {
+        if (config.matches(exchange.getRequest().getURI())) {
+            exchange.getResponse().beforeCommit(() -> {
+                return log(exchange);
+            });
+        }
+
+        return chain.filter(exchange);
+    }
+
+    private static final AnonymousAuthenticationToken ANNON = new AnonymousAuthenticationToken("anonymous", "anonymous",
+            List.of(new SimpleGrantedAuthority("ROLE_ANONYMOUS")));
+
+    /**
+     * @param exchange
+     */
+    private Mono<Void> log(ServerWebExchange exchange) {
+        if (!log.isInfoEnabled())
+            return Mono.empty();
+
+        return exchange.getPrincipal().switchIfEmpty(Mono.just(ANNON)).doOnNext(p -> {
+            doLog(p, exchange);
+        }).then();
+    }
+
+    private void doLog(Principal principal, ServerWebExchange exchange) {
+        ServerHttpRequest request = exchange.getRequest();
+        ServerHttpResponse response = exchange.getResponse();
+        URI uri = request.getURI();
+
+        Route route = exchange.getAttribute(GATEWAY_ROUTE_ATTR);
+        URI routeUri = exchange.getAttribute(GATEWAY_REQUEST_URL_ATTR);
+
+        String requestId = request.getHeaders().getFirst(RequestIdGlobalFilter.REQUEST_ID_HEADER);
+
+        InetSocketAddress addr = request.getRemoteAddress();
+        String remoteAddress = addr == null ? "unknown" : addr.toString();
+
+        mdcPut("route-id", route.getId());
+        mdcPut("route-uri", String.valueOf(routeUri));
+        mdcPut(RequestIdGlobalFilter.REQUEST_ID_HEADER, requestId);
+        mdcPut("remoteAddress", remoteAddress);
+
+        Optional<GeorchestraUser> user = GeorchestraUsers.resolve(exchange);
+        user.ifPresentOrElse(gsu -> {
+            mdcPut("auth-user", gsu.getUsername());
+            mdcPut("auth-roles", gsu.getRoles().stream().collect(Collectors.joining(", ")));
+        }, () -> MDC.put("auth-user", "anonymous"));
+
+        if (principal instanceof Authentication && principal != ANNON) {
+            String roles = ((Authentication) principal).getAuthorities().stream().map(GrantedAuthority::getAuthority)
+                    .collect(Collectors.joining(", "));
+
+            mdcPut("principal-name", principal.getName());
+            mdcPut("principal-authorities", roles);
+        }
+
+        log.info("{} {} {} ", request.getMethodValue(), response.getRawStatusCode(), uri);
+        mdcClear();
+    }
+
+    void mdcPut(String key, String value) {
+        MDC.put(key, value);
+    }
+
+    void mdcClear() {
+        MDC.clear();
+    }
+}

gateway/src/main/java/org/georchestra/gateway/filter/global/AccessLogFilterConfig.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2022 by the geOrchestra PSC
+ *
+ * This file is part of geOrchestra.
+ *
+ * geOrchestra is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option)
+ * any later version.
+ *
+ * geOrchestra is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * geOrchestra.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package org.georchestra.gateway.filter.global;
+
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+import lombok.Data;
+
+/**
+ * Configuration to set white/black list over the request URL to determine if
+ * the access log filter will log an entry for it.
+ */
+@Data
+@ConfigurationProperties(prefix = "georchestra.gateway.accesslog")
+public class AccessLogFilterConfig {
+
+    /**
+     * Enable/disable the access log filter
+     */
+    private boolean enabled = true;
+
+    /**
+     * A list of java regular expressions applied to the request URL to include them
+     * from logging.
+     */
+    List<Pattern> include = new ArrayList<>();
+
+    /**
+     * A list of java regular expressions applied to the request URL to exclude them
+     * from logging. A request URL must pass all the include filters before being
+     * tested for exclusion. Useful to avoid flooding the logs with frequent
+     * non-important requests such as static resources (i.e. static images, etc).
+     */
+    List<Pattern> exclude = new ArrayList<>();
+
+    /**
+     * @param uri the origin URL (e.g. https://my.domain.com/geoserver/web/)
+     * @return {@code true} if disabled or an access log entry shall be logged for
+     *         this request
+     */
+    public boolean matches(URI uri) {
+        if (!enabled || (include.isEmpty() && exclude.isEmpty()))
+            return true;
+
+        String url = uri.toString();
+        return matches(url, include, true) && !matches(url, exclude, false);
+    }
+
+    private boolean matches(String url, List<Pattern> patterns, boolean fallbackIfEmpty) {
+        return (patterns == null || patterns.isEmpty()) ? fallbackIfEmpty
+                : patterns.stream().anyMatch(pattern -> pattern.matcher(url).matches());
+    }
+}

gateway/src/main/java/org/georchestra/gateway/filter/global/RequestIdGlobalFilter.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2022 by the geOrchestra PSC
+ *
+ * This file is part of geOrchestra.
+ *
+ * geOrchestra is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option)
+ * any later version.
+ *
+ * geOrchestra is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * geOrchestra.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.georchestra.gateway.filter.global;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.springframework.cloud.gateway.filter.GatewayFilterChain;
+import org.springframework.cloud.gateway.filter.GlobalFilter;
+import org.springframework.core.Ordered;
+import org.springframework.http.server.reactive.ServerHttpRequest;
+import org.springframework.http.server.reactive.ServerHttpResponse;
+import org.springframework.web.server.ServerWebExchange;
+
+import reactor.core.publisher.Mono;
+
+/**
+ * Makes sure both the request and response have the same
+ * {@literal X-Request-ID} header.
+ * <p>
+ * A new value is created for the header if not provided by the client.
+ */
+public class RequestIdGlobalFilter implements GlobalFilter, Ordered {
+
+    static final String REQUEST_ID_HEADER = "X-Request-ID";
+
+    /**
+     * @return {@link Ordered#HIGHEST_PRECEDENCE}
+     */
+    public @Override int getOrder() {
+        return Ordered.HIGHEST_PRECEDENCE;
+    }
+
+    /**
+     * Makes sure both the request and response have the same
+     * {@literal X-Request-ID} header.
+     * <p>
+     * A new value is created for the header if not provided by the client.
+     */
+    public @Override Mono<Void> filter(ServerWebExchange exchange, GatewayFilterChain chain) {
+
+        final String requestId;
+        final ServerHttpRequest request;
+        String providedRequestId = exchange.getRequest().getHeaders().getFirst(REQUEST_ID_HEADER);
+        if (null == providedRequestId) {
+            requestId = RandomStringUtils.randomNumeric(16);
+            request = exchange.getRequest().mutate().header(REQUEST_ID_HEADER, requestId).build();
+            exchange = exchange.mutate().request(request).build();
+        } else {
+            requestId = providedRequestId;
+            request = exchange.getRequest();
+        }
+
+        ServerHttpResponse response = exchange.getResponse();
+        response.beforeCommit(() -> {
+            response.getHeaders().set(REQUEST_ID_HEADER, requestId);
+            return Mono.empty();
+        });
+
+        return chain.filter(exchange);
+    }
+
+}