Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Issue 29711 analytics matcher layer #29755

Merged
merged 36 commits into from
Sep 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
cf827da
Revert "fix: #28563 removing template ajax (#28572)"
jdotcms May 15, 2024
fb39637
Merge branch 'master' of github.com:dotCMS/core
jdotcms Jun 11, 2024
af6fed1
Merge branch 'master' of github.com:dotCMS/core
jdotcms Jun 14, 2024
96dd743
Merge branch 'master' of github.com:dotCMS/core
jdotcms Jun 18, 2024
fb71cf9
Merge branch 'master' of github.com:dotCMS/core
jdotcms Jun 24, 2024
ae1d5e1
Merge branches 'master' and 'master' of github.com:dotCMS/core
jdotcms Jun 25, 2024
b523af6
Merge branch 'master' of github.com:dotCMS/core
jdotcms Jun 27, 2024
95de159
Merge branch 'master' of github.com:dotCMS/core
jdotcms Jun 27, 2024
679b600
Merge branch 'master' of github.com:dotCMS/core
jdotcms Jun 28, 2024
4bb207b
Merge branch 'master' of github.com:dotCMS/core
jdotcms Jul 2, 2024
320b4b1
Merge branches 'master' and 'master' of github.com:dotCMS/core
jdotcms Jul 2, 2024
ed6a751
Merge branches 'master' and 'master' of github.com:dotCMS/core
jdotcms Jul 3, 2024
02a6e96
Merge branch 'master' of github.com:dotCMS/core
jdotcms Jul 4, 2024
5479786
Merge branch 'master' of github.com:dotCMS/core
jdotcms Jul 5, 2024
0762192
Merge branch 'master' of github.com:dotCMS/core
jdotcms Jul 9, 2024
2a61c7f
Merge branch 'master' of github.com:dotCMS/core
jdotcms Jul 11, 2024
0afad59
Merge branch 'master' of github.com:dotCMS/core
jdotcms Jul 17, 2024
9c510e7
Merge branch 'master' of github.com:dotCMS/core
jdotcms Jul 18, 2024
24a5827
Merge branch 'master' of github.com:dotCMS/core
jdotcms Jul 19, 2024
7468cd7
Merge branch 'master' of github.com:dotCMS/core
jdotcms Jul 22, 2024
0388022
Merge branch 'master' of github.com:dotCMS/core
jdotcms Jul 29, 2024
162ade1
Merge branch 'master' of github.com:dotCMS/core
jdotcms Aug 19, 2024
1e9be41
Merge branch 'master' of github.com:dotCMS/core
jdotcms Aug 21, 2024
17cac16
Merge branch 'master' of github.com:dotCMS/core
jdotcms Aug 21, 2024
ff615f0
Merge branch 'master' of github.com:dotCMS/core
jdotcms Aug 27, 2024
e2f47be
#29711 adding first draft for matcher layer
jdotcms Aug 27, 2024
c50e2c0
#29711 adding first draft for matcher layer
jdotcms Aug 27, 2024
1a8af52
#29711 getting closer to the matcher goal
jdotcms Aug 27, 2024
9e726ef
#29711 getting closer to the matcher goal
jdotcms Aug 27, 2024
ebec8f7
#29711 getting closer to the matcher goal
jdotcms Aug 27, 2024
a9a2243
#29711 getting closer to the matcher goal
jdotcms Aug 27, 2024
ba9fe4a
#29711 getting closer to the matcher goal
jdotcms Aug 27, 2024
df89ff7
#29711 getting closer to the matcher goal
jdotcms Aug 28, 2024
a6b6356
#29711 getting closer to the matcher goal
jdotcms Aug 29, 2024
ced4826
#29711 adding test fix
jdotcms Sep 3, 2024
3632da0
Merge branch 'master' into issue-29711-analytics-matcher-layer
jdotcms Sep 3, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package com.dotcms.analytics.track;

import com.dotcms.filters.interceptor.Result;
import com.dotcms.filters.interceptor.WebInterceptor;
import com.dotcms.jitsu.EventLogSubmitter;
import com.dotcms.util.CollectionsUtils;
import com.dotcms.util.WhiteBlackList;
import com.dotmarketing.util.Config;
import com.dotmarketing.util.Logger;
import com.liferay.util.StringPool;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Predicate;

/**
* Web Interceptor to track analytics
* @author jsanca
*/
public class AnalyticsTrackWebInterceptor implements WebInterceptor {

private final static Map<String, RequestMatcher> requestMatchersMap = new ConcurrentHashMap<>();

private final EventLogSubmitter submitter;

/// private static final String[] DEFAULT_BLACKLISTED_PROPS = new String[]{"^/api/*"};
private static final String[] DEFAULT_BLACKLISTED_PROPS = new String[]{StringPool.BLANK};
private final WhiteBlackList whiteBlackList = new WhiteBlackList.Builder()
.addWhitePatterns(Config.getStringArrayProperty("ANALYTICS_WHITELISTED_KEYS",
new String[]{StringPool.BLANK})) // allows everything
.addBlackPatterns(CollectionsUtils.concat(Config.getStringArrayProperty( // except this
"ANALYTICS_BLACKLISTED_KEYS", new String[]{}), DEFAULT_BLACKLISTED_PROPS)).build();

public AnalyticsTrackWebInterceptor() {

submitter = new EventLogSubmitter();
addRequestMatcher(
new PagesAndUrlMapsRequestMatcher(),
new FilesRequestMatcher(),
new RulesRedirectsRequestMatcher(),
new VanitiesRequestMatcher());
}

/**
* Add a request matchers
* @param requestMatchers
*/
public static void addRequestMatcher(final RequestMatcher... requestMatchers) {
for (final RequestMatcher matcher : requestMatchers) {
requestMatchersMap.put(matcher.getId(), matcher);
}
}

/**
* Remove a request matcher by id
* @param requestMatcherId
*/
public static void removeRequestMatcher(final String requestMatcherId) {

requestMatchersMap.remove(requestMatcherId);
}

@Override
public Result intercept(final HttpServletRequest request, final HttpServletResponse response) throws IOException {

if (whiteBlackList.isAllowed(request.getRequestURI())) {
final Optional<RequestMatcher> matcherOpt = this.anyMatcher(request, response, RequestMatcher::runBeforeRequest);
if (matcherOpt.isPresent()) {

Logger.debug(this, () -> "intercept, Matched: " + matcherOpt.get().getId() + " request: " + request.getRequestURI());
//fireNextStep(request, response);
}
}

return Result.NEXT;
}

@Override
public boolean afterIntercept(final HttpServletRequest request, final HttpServletResponse response) {

if (whiteBlackList.isAllowed(request.getRequestURI())) {
final Optional<RequestMatcher> matcherOpt = this.anyMatcher(request, response, RequestMatcher::runAfterRequest);
if (matcherOpt.isPresent()) {

Logger.debug(this, () -> "afterIntercept, Matched: " + matcherOpt.get().getId() + " request: " + request.getRequestURI());
//fireNextStep(request, response);
jdotcms marked this conversation as resolved.
Show resolved Hide resolved
}
}

return true;
}

private Optional<RequestMatcher> anyMatcher(final HttpServletRequest request, final HttpServletResponse response, Predicate<? super RequestMatcher> filterRequest) {

return requestMatchersMap.values().stream()
.filter(filterRequest)
.filter(matcher -> matcher.match(request, response))
.findFirst();
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package com.dotcms.analytics.track;

import com.dotcms.visitor.filter.characteristics.Character;
import com.dotcms.visitor.filter.characteristics.CharacterWebAPI;
import com.dotmarketing.business.web.WebAPILocator;
import com.dotmarketing.filters.CMSFilter;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.Objects;

/**
* Matcher for pages or files
* @author jsanca
*/
jdotcms marked this conversation as resolved.
Show resolved Hide resolved
public class FilesRequestMatcher implements RequestMatcher {

private final CharacterWebAPI characterWebAPI;

public FilesRequestMatcher() {
this(WebAPILocator.getCharacterWebAPI());
}

public FilesRequestMatcher(final CharacterWebAPI characterWebAPI) {
this.characterWebAPI = characterWebAPI;
}

@Override
public boolean runBeforeRequest() {
return true;
}

@Override
public boolean match(final HttpServletRequest request, final HttpServletResponse response) {

final Character character = this.characterWebAPI.getOrCreateCharacter(request, response);
if (Objects.nonNull(character)) {

final CMSFilter.IAm iAm = (CMSFilter.IAm) character.getMap().
getOrDefault("iAm", CMSFilter.IAm.NOTHING_IN_THE_CMS);

// should we have a fallback when nothing is returned???
return iAm == CMSFilter.IAm.FILE;
}

return false;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package com.dotcms.analytics.track;

import com.dotcms.visitor.filter.characteristics.Character;
import com.dotcms.visitor.filter.characteristics.CharacterWebAPI;
import com.dotmarketing.business.web.WebAPILocator;
import com.dotmarketing.filters.CMSFilter;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.Objects;

/**
* Matcher for pages or files
* @author jsanca
*/
public class PagesAndUrlMapsRequestMatcher implements RequestMatcher {
jdotcms marked this conversation as resolved.
Show resolved Hide resolved

private final CharacterWebAPI characterWebAPI;

public PagesAndUrlMapsRequestMatcher() {
this(WebAPILocator.getCharacterWebAPI());
}

public PagesAndUrlMapsRequestMatcher(final CharacterWebAPI characterWebAPI) {
this.characterWebAPI = characterWebAPI;
}

@Override
public boolean runBeforeRequest() {
return true;
}

@Override
public boolean match(final HttpServletRequest request, final HttpServletResponse response) {

final Character character = this.characterWebAPI.getOrCreateCharacter(request, response);
if (Objects.nonNull(character)) {

final CMSFilter.IAm iAm = (CMSFilter.IAm) character.getMap().
getOrDefault("iAm", CMSFilter.IAm.NOTHING_IN_THE_CMS);

// should we have a fallback when nothing is returned???
return iAm == CMSFilter.IAm.PAGE; // this captures also url maps
}

return false;
}
}
126 changes: 126 additions & 0 deletions dotCMS/src/main/java/com/dotcms/analytics/track/RequestMatcher.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
package com.dotcms.analytics.track;

import com.dotmarketing.util.Config;
import com.dotmarketing.util.RegEX;
import io.vavr.control.Try;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.util.Objects;
import java.util.Set;

/**
* Matcher to include the tracking for analytics of some request.
*
* @author jsanca
*/
public interface RequestMatcher {


String CHARSET = Try.of(()->Config.getStringProperty("CHARSET", "UTF-8")).getOrElse("UTF-8");

/**
* Return true if match the request with the patterns and methods
* @param request {@link HttpServletRequest}
* @param response {@link HttpServletResponse}
* @return boolean true if the request match the patterns and methods
*/
default boolean match(final HttpServletRequest request, final HttpServletResponse response) {

final Set<String> patterns = getMatcherPatterns();
final Set<String> methods = getAllowedMethods();
return Objects.nonNull(patterns) && !patterns.isEmpty() &&
Objects.nonNull(methods) && !methods.isEmpty() &&
isAllowedMethod (methods, request.getMethod()) &&
match(request, response, patterns);
}

/**
* Match the request with the patterns
* @param request {@link HttpServletRequest}
* @param response {@link HttpServletResponse}
* @param patterns Set of patterns
* @return boolean true if any of the patterns match the request
*/
default boolean match(final HttpServletRequest request, final HttpServletResponse response, final Set<String> patterns) {

final String requestURI = request.getRequestURI();
return patterns.stream().anyMatch(pattern -> match(requestURI, pattern));
}

/**
* Means the matcher should run before request
* @return boolean
*/
default boolean runBeforeRequest() {
return false;
}

/**
* Means the matcher should run after request
* @return boolean
*/
default boolean runAfterRequest() {
return false;
}
/**
* Match the request URI with the pattern
* @param requestURI String
* @param pattern String
* @return boolean true if the pattern match the request URI
*/
default boolean match (final String requestURI, final String pattern) {

String uftUri = null;

try {

uftUri = URLDecoder.decode(requestURI, CHARSET);
} catch (UnsupportedEncodingException e) {

uftUri = requestURI;
}

return RegEX.containsCaseInsensitive(uftUri, pattern.trim());
jdotcms marked this conversation as resolved.
Show resolved Hide resolved
} // match.

/**
* Determinate if the method is allowed
* @param methods Set of methods
* @param method String current request method
* @return boolean true if the method is allowed
*/
default boolean isAllowedMethod(final Set<String> methods, final String method) {

return methods.contains(method);
}

/**
* Returns a set of patterns for the matcher
* @return Set by default empty
*/
default Set<String> getMatcherPatterns() {

return Set.of();
}

/**
* Returns the request methods allowed for this matcher.
* @return Set by default empty
*/
default Set<String> getAllowedMethods() {

return Set.of();
}

/**
* Return an id for the Matcher, by default returns the class name.
* @return
*/
default String getId() {

return this.getClass().getName();
}
}
jdotcms marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package com.dotcms.analytics.track;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.Objects;

/**
* Matcher for vanity urls and rules redirect
* @author jsanca
*/
public class RulesRedirectsRequestMatcher implements RequestMatcher {

@Override
public boolean runAfterRequest() {
return true;
}

@Override
public boolean match(final HttpServletRequest request, final HttpServletResponse response) {

final String ruleRedirect = response.getHeader("X-DOT-SendRedirectRuleAction");
return Objects.nonNull(ruleRedirect) && "true".equals(ruleRedirect);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package com.dotcms.analytics.track;

import com.dotmarketing.filters.Constants;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.Objects;

/**
* Matcher for vanity urls and rules redirect
* @author jsanca
*/
public class VanitiesRequestMatcher implements RequestMatcher {

@Override
public boolean runAfterRequest() {
return true;
}

@Override
public boolean match(final HttpServletRequest request, final HttpServletResponse response) {

final Object vanityHasRun = request.getAttribute(Constants.VANITY_URL_OBJECT);

return Objects.nonNull(vanityHasRun);
}
}
Loading
Loading