Workspace termination time metrics

deploy/openshift/templates/monitoring/grafana-dashboards.yaml

@@ -4287,6 +4287,95 @@ data:
             "x": 6,
             "y": 12
           },
+          "id": 49,
+          "legend": {
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": false,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null as zero",
+          "options": {},
+          "paceLength": 10,
+          "percentage": false,
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(che_workspace_stop_time_seconds_sum[5m])/rate(che_workspace_stop_time_seconds_count [5m])",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "{{result}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Average workspace stop time",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "s",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fill": 1,
+          "gridPos": {
+            "h": 5,
+            "w": 6,
+            "x": 12,
+            "y": 12
+          },
           "id": 8,
           "legend": {
             "avg": false,

wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceStopTrackerMeterBinder.java

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2012-2018 Red Hat, Inc.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *   Red Hat, Inc. - initial API and implementation
+ */
+package org.eclipse.che.api.metrics;
+
+import static org.eclipse.che.api.core.model.workspace.WorkspaceStatus.*;
+import static org.eclipse.che.api.metrics.WorkspaceBinders.withStandardTags;
+import static org.eclipse.che.api.metrics.WorkspaceBinders.workspaceMetric;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.inject.Inject;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Timer;
+import io.micrometer.core.instrument.binder.MeterBinder;
+import java.time.Duration;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import javax.inject.Singleton;
+import org.eclipse.che.api.core.notification.EventService;
+import org.eclipse.che.api.workspace.shared.dto.event.WorkspaceStatusEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * {@link MeterBinder} that is providing metrics about workspace stop time.
+ *
+ * <p>We're measuring these state transitions:
+ *
+ * <pre>
+ *   RUNNING -> STOPPING -> STOPPED
+ *   STARTING -> STOPPING -> STOPPED
+ * </pre>
+ */
+@Singleton
+public class WorkspaceStopTrackerMeterBinder implements MeterBinder {
+
+  private static final Logger LOG = LoggerFactory.getLogger(WorkspaceStopTrackerMeterBinder.class);
+
+  private final EventService eventService;
+
+  private final Map<String, Long> workspaceStopTime;
+  private Timer stopTimer;
+
+  @Inject
+  public WorkspaceStopTrackerMeterBinder(EventService eventService) {
+    this(eventService, new ConcurrentHashMap<>());
+  }
+
+  @VisibleForTesting
+  WorkspaceStopTrackerMeterBinder(EventService eventService, Map<String, Long> workspaceStopTime) {
+    this.eventService = eventService;
+    this.workspaceStopTime = workspaceStopTime;
+  }
+
+  @Override
+  public void bindTo(MeterRegistry registry) {
+
+    stopTimer =
+        Timer.builder(workspaceMetric("stop.time"))
+            .description("The time of workspace stop")
+            .tags(withStandardTags("result", "success"))
+            .register(registry);
+
+    // only subscribe to the event once we have the counters ready
+    eventService.subscribe(this::handleWorkspaceStatusChange, WorkspaceStatusEvent.class);
+  }
+
+  private void handleWorkspaceStatusChange(WorkspaceStatusEvent event) {
+    if ((event.getPrevStatus() == RUNNING || event.getPrevStatus() == STARTING)
+        && event.getStatus() == STOPPING) {
+      workspaceStopTime.put(event.getWorkspaceId(), System.currentTimeMillis());
+    } else if (event.getPrevStatus() == STOPPING) {
+      Long stopTime = workspaceStopTime.remove(event.getWorkspaceId());
+      if (stopTime == null) {
+        LOG.warn("No workspace stop time recorded for workspace {}", event.getWorkspaceId());
+        return;
+      }
+
+      if (event.getStatus() == STOPPED) {
+        stopTimer.record(Duration.ofMillis(System.currentTimeMillis() - stopTime));
+      } else {
+        LOG.error("Unexpected change of status from STOPPING to {}", event.getStatus());
+        return;
+      }
+    }
+  }
+}

wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WsMasterMetricsModule.java

@@ -29,6 +29,7 @@ protected void configure() {
     meterMultibinder.addBinding().to(WorkspaceActivityMeterBinder.class);
     meterMultibinder.addBinding().to(WorkspaceFailureMeterBinder.class);
     meterMultibinder.addBinding().to(WorkspaceStartTrackerMeterBinder.class);
+    meterMultibinder.addBinding().to(WorkspaceStopTrackerMeterBinder.class);
     meterMultibinder.addBinding().to(WorkspaceSuccessfulStartAttemptsMeterBinder.class);
     meterMultibinder.addBinding().to(WorkspaceSuccessfulStopAttemptsMeterBinder.class);
     meterMultibinder.addBinding().to(WorkspaceStartAttemptsMeterBinder.class);

wsmaster/che-core-api-metrics/src/test/java/org/eclipse/che/api/metrics/WorkspaceStartTrackerMeterBinderTest.java

@@ -19,7 +19,6 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.Random;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import org.eclipse.che.api.core.model.workspace.WorkspaceStatus;
@@ -33,7 +32,6 @@
 
 public class WorkspaceStartTrackerMeterBinderTest {
 
-  private Random rnd = new Random();
   private EventService eventService;
   private MeterRegistry registry;
   private WorkspaceStartTrackerMeterBinder meterBinder;

wsmaster/che-core-api-metrics/src/test/java/org/eclipse/che/api/metrics/WorkspaceStopTrackerMeterBinderTest.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2012-2018 Red Hat, Inc.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *   Red Hat, Inc. - initial API and implementation
+ */
+package org.eclipse.che.api.metrics;
+
+import static java.util.Arrays.asList;
+
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Timer;
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import org.eclipse.che.api.core.model.workspace.WorkspaceStatus;
+import org.eclipse.che.api.core.notification.EventService;
+import org.eclipse.che.api.workspace.shared.dto.event.WorkspaceStatusEvent;
+import org.eclipse.che.dto.server.DtoFactory;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class WorkspaceStopTrackerMeterBinderTest {
+
+  private EventService eventService;
+  private MeterRegistry registry;
+  private WorkspaceStopTrackerMeterBinder meterBinder;
+  private Map<String, Long> workspaceStopTime;
+
+  @BeforeMethod
+  public void setUp() {
+    eventService = new EventService();
+    registry = new SimpleMeterRegistry();
+    workspaceStopTime = new ConcurrentHashMap<>();
+    meterBinder = new WorkspaceStopTrackerMeterBinder(eventService, workspaceStopTime);
+    meterBinder.bindTo(registry);
+  }
+
+  @Test
+  public void shouldRecordWorkspaceStopTime() {
+    // given
+    // when
+    eventService.publish(
+        DtoFactory.newDto(WorkspaceStatusEvent.class)
+            .withPrevStatus(WorkspaceStatus.RUNNING)
+            .withStatus(WorkspaceStatus.STOPPING)
+            .withWorkspaceId("id1"));
+    // then
+    Assert.assertTrue(workspaceStopTime.containsKey("id1"));
+  }
+
+  @Test(dataProvider = "allStatusTransitionsWithoutStarting")
+  public void shouldNotRecordWorkspaceStopTimeForNonStoppingStatuses(
+      WorkspaceStatus from, WorkspaceStatus to) {
+    // given
+    // when
+    eventService.publish(
+        DtoFactory.newDto(WorkspaceStatusEvent.class)
+            .withPrevStatus(from)
+            .withStatus(to)
+            .withWorkspaceId("id1"));
+    // then
+    Assert.assertTrue(workspaceStopTime.isEmpty());
+  }
+
+  @Test
+  public void shouldCountSuccessfulStart() {
+    // given
+    workspaceStopTime.put("id1", System.currentTimeMillis() - 60 * 1000);
+    // when
+    eventService.publish(
+        DtoFactory.newDto(WorkspaceStatusEvent.class)
+            .withPrevStatus(WorkspaceStatus.STOPPING)
+            .withStatus(WorkspaceStatus.STOPPED)
+            .withWorkspaceId("id1"));
+
+    // then
+
+    Timer t = registry.find("che.workspace.stop.time").tag("result", "success").timer();
+    Assert.assertEquals(t.count(), 1);
+    Assert.assertTrue(t.totalTime(TimeUnit.MILLISECONDS) >= 60 * 1000);
+  }
+
+  @DataProvider
+  public Object[][] allStatusTransitionsWithoutStarting() {
+    List<List<WorkspaceStatus>> transitions = new ArrayList<>(9);
+
+    for (WorkspaceStatus from : WorkspaceStatus.values()) {
+      for (WorkspaceStatus to : WorkspaceStatus.values()) {
+        if ((from == WorkspaceStatus.RUNNING || from == WorkspaceStatus.STARTING)
+            && to == WorkspaceStatus.STOPPING) {
+          continue;
+        }
+
+        transitions.add(asList(from, to));
+      }
+    }
+
+    return transitions.stream().map(List::toArray).toArray(Object[][]::new);
+  }
+}