add HC detector request/failure stats

src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorPlugin.java

@@ -435,6 +435,8 @@ public Collection<Object> createComponents(
             .<String, ADStat<?>>builder()
             .put(StatNames.AD_EXECUTE_REQUEST_COUNT.getName(), new ADStat<>(false, new CounterSupplier()))
             .put(StatNames.AD_EXECUTE_FAIL_COUNT.getName(), new ADStat<>(false, new CounterSupplier()))
+            .put(StatNames.AD_HC_EXECUTE_REQUEST_COUNT.getName(), new ADStat<>(false, new CounterSupplier()))
+            .put(StatNames.AD_HC_EXECUTE_FAIL_COUNT.getName(), new ADStat<>(false, new CounterSupplier()))
             .put(StatNames.MODEL_INFORMATION.getName(), new ADStat<>(false, new ModelsOnNodeSupplier(modelManager, cacheProvider)))
             .put(
                 StatNames.ANOMALY_DETECTORS_INDEX_STATUS.getName(),

src/main/java/com/amazon/opendistroforelasticsearch/ad/stats/StatNames.java

@@ -24,6 +24,8 @@
 public enum StatNames {
     AD_EXECUTE_REQUEST_COUNT("ad_execute_request_count"),
     AD_EXECUTE_FAIL_COUNT("ad_execute_failure_count"),
+    AD_HC_EXECUTE_REQUEST_COUNT("ad_hc_execute_request_count"),
+    AD_HC_EXECUTE_FAIL_COUNT("ad_hc_execute_failure_count"),
     DETECTOR_COUNT("detector_count"),
     ANOMALY_DETECTORS_INDEX_STATUS("anomaly_detectors_index_status"),
     ANOMALY_RESULTS_INDEX_STATUS("anomaly_results_index_status"),

src/main/java/com/amazon/opendistroforelasticsearch/ad/transport/AnomalyResultTransportAction.java

@@ -17,6 +17,7 @@
 
 import java.net.ConnectException;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -123,6 +124,9 @@ public class AnomalyResultTransportAction extends HandledTransportAction<ActionR
     private final Client client;
     private final SearchFeatureDao searchFeatureDao;
 
+    // cache HC detector id
+    private final Set<String> hcDetectors;
+
     @Inject
     public AnomalyResultTransportAction(
         ActionFilters actionFilters,
@@ -160,6 +164,7 @@ public AnomalyResultTransportAction(
         this.adStats = adStats;
         this.threadPool = threadPool;
         this.searchFeatureDao = searchFeatureDao;
+        this.hcDetectors = new HashSet<>();
     }
 
     /**
@@ -217,14 +222,20 @@ public AnomalyResultTransportAction(
     protected void doExecute(Task task, ActionRequest actionRequest, ActionListener<AnomalyResultResponse> listener) {
         try (ThreadContext.StoredContext context = client.threadPool().getThreadContext().stashContext()) {
             AnomalyResultRequest request = AnomalyResultRequest.fromActionRequest(actionRequest);
+            String adID = request.getAdID();
             ActionListener<AnomalyResultResponse> original = listener;
-            listener = ActionListener.wrap(original::onResponse, e -> {
+            listener = ActionListener.wrap(r -> {
+                hcDetectors.remove(adID);
+                original.onResponse(r);
+            }, e -> {
                 adStats.getStat(StatNames.AD_EXECUTE_FAIL_COUNT.getName()).increment();
+                if (hcDetectors.contains(adID)) {
+                    adStats.getStat(StatNames.AD_HC_EXECUTE_FAIL_COUNT.getName()).increment();
+                }
+                hcDetectors.remove(adID);
                 original.onFailure(e);
             });
 
-            String adID = request.getAdID();
-
             if (!EnabledSetting.isADPluginEnabled()) {
                 throw new EndRunException(adID, CommonErrorMessages.DISABLED_ERR_MSG, true);
             }
@@ -258,6 +269,10 @@ private ActionListener<Optional<AnomalyDetector>> onGetDetector(
             }
 
             AnomalyDetector anomalyDetector = detectorOptional.get();
+            if (anomalyDetector.isMultientityDetector()) {
+                hcDetectors.add(adID);
+                adStats.getStat(StatNames.AD_HC_EXECUTE_REQUEST_COUNT.getName()).increment();
+            }
 
             long delayMillis = Optional
                 .ofNullable((IntervalTimeConfiguration) anomalyDetector.getWindowDelay())

src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/AnomalyResultTests.java

@@ -272,6 +272,8 @@ public void setUp() throws Exception {
             {
                 put(StatNames.AD_EXECUTE_REQUEST_COUNT.getName(), new ADStat<>(false, new CounterSupplier()));
                 put(StatNames.AD_EXECUTE_FAIL_COUNT.getName(), new ADStat<>(false, new CounterSupplier()));
+                put(StatNames.AD_HC_EXECUTE_REQUEST_COUNT.getName(), new ADStat<>(false, new CounterSupplier()));
+                put(StatNames.AD_HC_EXECUTE_FAIL_COUNT.getName(), new ADStat<>(false, new CounterSupplier()));
             }
         };
 

src/test/java/com/amazon/opendistroforelasticsearch/ad/transport/MultientityResultTests.java

@@ -186,6 +186,8 @@ public void setUp() throws Exception {
             {
                 put(StatNames.AD_EXECUTE_REQUEST_COUNT.getName(), new ADStat<>(false, new CounterSupplier()));
                 put(StatNames.AD_EXECUTE_FAIL_COUNT.getName(), new ADStat<>(false, new CounterSupplier()));
+                put(StatNames.AD_HC_EXECUTE_REQUEST_COUNT.getName(), new ADStat<>(false, new CounterSupplier()));
+                put(StatNames.AD_HC_EXECUTE_FAIL_COUNT.getName(), new ADStat<>(false, new CounterSupplier()));
             }
         };
         adStats = new ADStats(indexUtils, normalModelManager, statsMap);