#29711 adding the collectors and refactor

dotCMS/src/main/java/com/dotcms/analytics/track/AnalyticsTrackWebInterceptor.java

@@ -1,13 +1,32 @@
 package com.dotcms.analytics.track;
 
+import com.dotcms.analytics.track.collectors.CharacterCollectorContextMap;
+import com.dotcms.analytics.track.collectors.Collector;
+import com.dotcms.analytics.track.collectors.CollectorContextMap;
+import com.dotcms.analytics.track.collectors.CollectorPayloadBean;
+import com.dotcms.analytics.track.collectors.ConcurrentCollectorPayloadBean;
+import com.dotcms.analytics.track.collectors.RequestCharacterCollectorContextMap;
+import com.dotcms.analytics.track.matchers.FilesRequestMatcher;
+import com.dotcms.analytics.track.matchers.PagesAndUrlMapsRequestMatcher;
+import com.dotcms.analytics.track.matchers.RequestMatcher;
+import com.dotcms.analytics.track.matchers.RulesRedirectsRequestMatcher;
+import com.dotcms.analytics.track.matchers.VanitiesRequestMatcher;
+import com.dotcms.concurrent.DotConcurrentFactory;
 import com.dotcms.filters.interceptor.Result;
 import com.dotcms.filters.interceptor.WebInterceptor;
+import com.dotcms.http.CircuitBreakerUrlBuilder;
+import com.dotcms.jitsu.EventLogRunnable;
 import com.dotcms.jitsu.EventLogSubmitter;
+import com.dotcms.jitsu.EventsPayload;
 import com.dotcms.util.CollectionsUtils;
 import com.dotcms.util.WhiteBlackList;
+import com.dotcms.visitor.filter.characteristics.Character;
+import com.dotmarketing.beans.Host;
+import com.dotmarketing.business.web.WebAPILocator;
 import com.dotmarketing.util.Config;
 import com.dotmarketing.util.Logger;
 import com.liferay.util.StringPool;
+import org.apache.http.HttpStatus;
 
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
@@ -24,6 +43,8 @@
 public class AnalyticsTrackWebInterceptor implements WebInterceptor {
 
  private final static Map<String, RequestMatcher> requestMatchersMap = new ConcurrentHashMap<>();
+ private final static Map<String, Collector> syncCollectors = new ConcurrentHashMap<>();
+ private final static Map<String, Collector> asyncCollectors = new ConcurrentHashMap<>();
 
  private final EventLogSubmitter submitter;
 
@@ -64,6 +85,36 @@ public static void removeRequestMatcher(final String requestMatcherId) {
  requestMatchersMap.remove(requestMatcherId);
  }
 
+ /**
+ * Add a collector
+ * @param collectors
+ */
+ public static void addCollector(final Collector... collectors) {
+ for (final Collector collector : collectors) {
+ if (collector.isAsync()) {
+
+ asyncCollectors.put(collector.getId(), collector);
+ } else {
+ syncCollectors.put(collector.getId(), collector);
+ }
+ }
+ }
+
+ /**
+ * Remove a collector by id
+ * @param collectorId
+ */
+ public static void removeCollector(final String collectorId) {
+
+ if (syncCollectors.containsKey(collectorId)) {
+ syncCollectors.remove(collectorId);
+ }
+
+ if (asyncCollectors.containsKey(collectorId)) {
+ asyncCollectors.remove(collectorId);
+ }
+ }
+
  @Override
  public Result intercept(final HttpServletRequest request, final HttpServletResponse response) throws IOException {
 
@@ -72,7 +123,7 @@ public Result intercept(final HttpServletRequest request, final HttpServletRespo
  if (matcherOpt.isPresent()) {
 
  Logger.debug(this, () -> "intercept, Matched: " + matcherOpt.get().getId() + " request: " + request.getRequestURI());
- //fireNextStep(request, response);
+ fireNext(request, response, matcherOpt.get());
  }
  }
 
@@ -87,7 +138,7 @@ public boolean afterIntercept(final HttpServletRequest request, final HttpServle
  if (matcherOpt.isPresent()) {
 
  Logger.debug(this, () -> "afterIntercept, Matched: " + matcherOpt.get().getId() + " request: " + request.getRequestURI());
- //fireNextStep(request, response);
+ fireNext(request, response, matcherOpt.get());
  }
  }
 
@@ -102,4 +153,50 @@ private Optional<RequestMatcher> anyMatcher(final HttpServletRequest request, fi
  .findFirst();
  }
 
+ /**
+ * Since the Fire the next step on the Analytics pipeline
+ * @param request
+ * @param response
+ * @param requestMatcher
+ */
+ protected void fireNext(final HttpServletRequest request, final HttpServletResponse response,
+ final RequestMatcher requestMatcher) {
+
+ Logger.debug(this, ()-> "fireNext, uri: " + request.getRequestURI() +
+ " requestMatcher: " + requestMatcher.getId());
+
+ if (!asyncCollectors.isEmpty() || !syncCollectors.isEmpty()) {
+
+ final CollectorPayloadBean collectorPayloadBean = new ConcurrentCollectorPayloadBean();
+ this.runCollectors(request, response, requestMatcher, collectorPayloadBean);
+ }
+ }
+
+ private void runCollectors(final HttpServletRequest request,
+ final HttpServletResponse response,
+ final RequestMatcher requestMatcher,
+ final CollectorPayloadBean collectorPayloadBean) {
+
+ final Character character = WebAPILocator.getCharacterWebAPI().getOrCreateCharacter(request, response);
+ final Host site = WebAPILocator.getHostWebAPI().getCurrentHostNoThrow(request);
+ if (!syncCollectors.isEmpty()) {
+
+ Logger.debug(this, ()-> "Running sync collectors");
+ final CollectorContextMap syncCollectorContextMap = new RequestCharacterCollectorContextMap(request, character, requestMatcher);
+ // we collect info which is sync and includes the request.
+ syncCollectors.values().forEach(collector -> collector.collect(syncCollectorContextMap, collectorPayloadBean));
+ }
+
+ // if there is anything to run async
+ final CollectorContextMap collectorContextMap = new CharacterCollectorContextMap(character, requestMatcher);
+ this.submitter.logEvent(
+ new EventLogRunnable(site, ()-> {
+ Logger.debug(this, ()-> "Running async collectors");
+ asyncCollectors.values().forEach(collector -> { collector.collect(collectorContextMap, collectorPayloadBean); });
+ return collectorPayloadBean.toMap();
+ }));
+
+ }
+
+
 }

dotCMS/src/main/java/com/dotcms/analytics/track/BasicProfileCollector.java → dotCMS/src/main/java/com/dotcms/analytics/track/collectors/BasicProfileCollector.java

@@ -1,4 +1,4 @@
-package com.dotcms.analytics.track;
+package com.dotcms.analytics.track.collectors;
 
 import com.dotcms.enterprise.cluster.ClusterFactory;
 import com.dotcms.util.FunctionUtils;

dotCMS/src/main/java/com/dotcms/analytics/track/collectors/CharacterCollectorContextMap.java

@@ -0,0 +1,41 @@
+package com.dotcms.analytics.track.collectors;
+
+import com.dotcms.analytics.track.matchers.RequestMatcher;
+import com.dotcms.visitor.filter.characteristics.Character;
+
+import javax.servlet.http.HttpServletRequest;
+
+/**
+ * This Context Map has the character map
+ * @author jsanca
+ */
+public class CharacterCollectorContextMap implements CollectorContextMap {
+
+ private final RequestMatcher requestMatcher;
+ private final Character character;
+
+ public CharacterCollectorContextMap(final Character character,
+ final RequestMatcher requestMatcher) {
+ this.character = character;
+ this.requestMatcher = requestMatcher;
+ }
+
+
+
+ @Override
+ public Object get(final String key) {
+
+
+ if (this.character.getMap().containsKey(key)) {
+ return this.character.getMap().get(key);
+ }
+
+ return null;
+ }
+
+
+ @Override
+ public RequestMatcher getRequestMatcher() {
+ return this.requestMatcher;
+ }
+}

dotCMS/src/main/java/com/dotcms/analytics/track/Collector.java → dotCMS/src/main/java/com/dotcms/analytics/track/collectors/Collector.java

@@ -1,4 +1,4 @@
-package com.dotcms.analytics.track;
+package com.dotcms.analytics.track.collectors;
 
 /**
  * A collector command basically puts information into a collector payload bean
@@ -27,4 +27,13 @@ public interface Collector {
  default boolean isAsync() {
  return false;
  }
+
+ /**
+ * Return an id for the Collector, by default returns the class name.
+ * @return
+ */
+ default String getId() {
+
+ return this.getClass().getName();
+ }
 }

dotCMS/src/main/java/com/dotcms/analytics/track/collectors/CollectorContextMap.java

@@ -0,0 +1,9 @@
+package com.dotcms.analytics.track.collectors;
+
+import com.dotcms.analytics.track.matchers.RequestMatcher;
+
+public interface CollectorContextMap {
+
+ Object get(String key);
+ RequestMatcher getRequestMatcher(); // since we do not have the previous step phase we need to keep this as an object, but will be a RequestMatcher
+}

dotCMS/src/main/java/com/dotcms/analytics/track/CollectorPayloadBean.java → dotCMS/src/main/java/com/dotcms/analytics/track/collectors/CollectorPayloadBean.java

@@ -1,6 +1,7 @@
-package com.dotcms.analytics.track;
+package com.dotcms.analytics.track.collectors;
 
 import java.io.Serializable;
+import java.util.Map;
 
 /**
  * Encapsulate the basic signature for a collector payload bean
@@ -10,4 +11,5 @@ public interface CollectorPayloadBean {
 
  CollectorPayloadBean put(String key, Serializable value);
  Serializable get(String key);
+ Map<String, Serializable> toMap();
 }

dotCMS/src/main/java/com/dotcms/analytics/track/ConcurrentCollectorPayloadBean.java → dotCMS/src/main/java/com/dotcms/analytics/track/collectors/ConcurrentCollectorPayloadBean.java

@@ -1,6 +1,7 @@
-package com.dotcms.analytics.track;
+package com.dotcms.analytics.track.collectors;
 
 import java.io.Serializable;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
 /**
@@ -22,4 +23,9 @@ public Serializable get(final String key) {
  return map.get(key);
  }
 
+ @Override
+ public Map<String, Serializable> toMap() {
+ return Map.copyOf(map);
+ }
+
 }

dotCMS/src/main/java/com/dotcms/analytics/track/collectors/FilesCollector.java

@@ -0,0 +1,74 @@
+package com.dotcms.analytics.track.collectors;
+
+import com.dotcms.rest.api.v1.DotObjectMapperProvider;
+import com.dotmarketing.beans.Host;
+import com.dotmarketing.business.APILocator;
+import com.dotmarketing.portlets.contentlet.business.HostAPI;
+import com.dotmarketing.portlets.fileassets.business.FileAsset;
+import com.dotmarketing.portlets.fileassets.business.FileAssetAPI;
+import io.vavr.control.Try;
+
+import java.io.StringWriter;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+public class FilesCollector implements Collector {
+
+ private final FileAssetAPI fileAssetAPI;
+ private final HostAPI hostAPI;
+
+
+ public FilesCollector() {
+ this(APILocator.getFileAssetAPI(),
+ APILocator.getHostAPI());
+ }
+
+ public FilesCollector(final FileAssetAPI fileAssetAPI,
+ final HostAPI hostAPI) {
+
+ this.fileAssetAPI = fileAssetAPI;
+ this.hostAPI = hostAPI;
+ }
+
+ @Override
+ public boolean test(CollectorContextMap collectorContextMap) {
+ return "file" == collectorContextMap.getRequestMatcher(); // should compare with the id
+ }
+
+
+ @Override
+ public CollectorPayloadBean collect(final CollectorContextMap collectorContextMap,
+ final CollectorPayloadBean collectorPayloadBean) {
+
+ final String uri = (String)collectorContextMap.get("uri");
+ final String siteId = (String)collectorContextMap.get("host");
+ final Long languageId = (Long)collectorContextMap.get("langId");
+ final String language = (String)collectorContextMap.get("lang");
+ final Map<String, String> pageObject = new HashMap<>();
+
+ if (Objects.nonNull(uri) && Objects.nonNull(siteId) && Objects.nonNull(languageId)) {
+
+ final Host site = Try.of(()->this.hostAPI.find(siteId, APILocator.systemUser(), false)).get();
+ final FileAsset fileAsset = Try.of(()->this.fileAssetAPI.getFileByPath(uri, site, languageId, true)).get();
+ pageObject.put("object_id", fileAsset.getIdentifier());
+ pageObject.put("title", fileAsset.getTitle());
+ pageObject.put("path", uri);
+ }
+
+ final StringWriter writer = new StringWriter();
+ Try.run(()-> DotObjectMapperProvider.getInstance().getDefaultObjectMapper().writeValue(writer, pageObject));
+ collectorPayloadBean.put("objects", writer.toString());
+ collectorPayloadBean.put("path", uri);
+ collectorPayloadBean.put("event", "file");
+ collectorPayloadBean.put("language", language);
+ collectorPayloadBean.put("site", siteId);
+
+ return collectorPayloadBean;
+ }
+
+ @Override
+ public boolean isAsync() {
+ return true;
+ }
+}

dotCMS/src/main/java/com/dotcms/analytics/track/PagesCollector.java → dotCMS/src/main/java/com/dotcms/analytics/track/collectors/PagesCollector.java

@@ -1,4 +1,4 @@
-package com.dotcms.analytics.track;
+package com.dotcms.analytics.track.collectors;
 
 import com.dotcms.rest.api.v1.DotObjectMapperProvider;
 import com.dotmarketing.beans.Host;