Skip to content

Commit

Permalink
Workspace termination time metrics (#13635)
Browse files Browse the repository at this point in the history
add workspace termination time metrics
Signed-off-by: Michal Vala <mvala@redhat.com>
  • Loading branch information
sparkoo authored and skabashnyuk committed Jul 2, 2019
1 parent b2dff13 commit cd891d7
Show file tree
Hide file tree
Showing 5 changed files with 296 additions and 2 deletions.
89 changes: 89 additions & 0 deletions deploy/openshift/templates/monitoring/grafana-dashboards.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4287,6 +4287,95 @@ data:
"x": 6,
"y": 12
},
"id": 49,
"legend": {
"avg": true,
"current": true,
"max": true,
"min": true,
"rightSide": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null as zero",
"options": {},
"paceLength": 10,
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(che_workspace_stop_time_seconds_sum[5m])/rate(che_workspace_stop_time_seconds_count [5m])",
"format": "time_series",
"hide": false,
"intervalFactor": 1,
"legendFormat": "{{result}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Average workspace stop time",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fill": 1,
"gridPos": {
"h": 5,
"w": 6,
"x": 12,
"y": 12
},
"id": 8,
"legend": {
"avg": false,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.api.metrics;

import static org.eclipse.che.api.core.model.workspace.WorkspaceStatus.*;
import static org.eclipse.che.api.metrics.WorkspaceBinders.withStandardTags;
import static org.eclipse.che.api.metrics.WorkspaceBinders.workspaceMetric;

import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Inject;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
import io.micrometer.core.instrument.binder.MeterBinder;
import java.time.Duration;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import javax.inject.Singleton;
import org.eclipse.che.api.core.notification.EventService;
import org.eclipse.che.api.workspace.shared.dto.event.WorkspaceStatusEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* {@link MeterBinder} that is providing metrics about workspace stop time.
*
* <p>We're measuring these state transitions:
*
* <pre>
* RUNNING -> STOPPING -> STOPPED
* STARTING -> STOPPING -> STOPPED
* </pre>
*/
@Singleton
public class WorkspaceStopTrackerMeterBinder implements MeterBinder {

private static final Logger LOG = LoggerFactory.getLogger(WorkspaceStopTrackerMeterBinder.class);

private final EventService eventService;

private final Map<String, Long> workspaceStopTime;
private Timer stopTimer;

@Inject
public WorkspaceStopTrackerMeterBinder(EventService eventService) {
this(eventService, new ConcurrentHashMap<>());
}

@VisibleForTesting
WorkspaceStopTrackerMeterBinder(EventService eventService, Map<String, Long> workspaceStopTime) {
this.eventService = eventService;
this.workspaceStopTime = workspaceStopTime;
}

@Override
public void bindTo(MeterRegistry registry) {

stopTimer =
Timer.builder(workspaceMetric("stop.time"))
.description("The time of workspace stop")
.tags(withStandardTags("result", "success"))
.register(registry);

// only subscribe to the event once we have the counters ready
eventService.subscribe(this::handleWorkspaceStatusChange, WorkspaceStatusEvent.class);
}

private void handleWorkspaceStatusChange(WorkspaceStatusEvent event) {
if ((event.getPrevStatus() == RUNNING || event.getPrevStatus() == STARTING)
&& event.getStatus() == STOPPING) {
workspaceStopTime.put(event.getWorkspaceId(), System.currentTimeMillis());
} else if (event.getPrevStatus() == STOPPING) {
Long stopTime = workspaceStopTime.remove(event.getWorkspaceId());
if (stopTime == null) {
LOG.warn("No workspace stop time recorded for workspace {}", event.getWorkspaceId());
return;
}

if (event.getStatus() == STOPPED) {
stopTimer.record(Duration.ofMillis(System.currentTimeMillis() - stopTime));
} else {
LOG.error("Unexpected change of status from STOPPING to {}", event.getStatus());
return;
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ protected void configure() {
meterMultibinder.addBinding().to(WorkspaceActivityMeterBinder.class);
meterMultibinder.addBinding().to(WorkspaceFailureMeterBinder.class);
meterMultibinder.addBinding().to(WorkspaceStartTrackerMeterBinder.class);
meterMultibinder.addBinding().to(WorkspaceStopTrackerMeterBinder.class);
meterMultibinder.addBinding().to(WorkspaceSuccessfulStartAttemptsMeterBinder.class);
meterMultibinder.addBinding().to(WorkspaceSuccessfulStopAttemptsMeterBinder.class);
meterMultibinder.addBinding().to(WorkspaceStartAttemptsMeterBinder.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import org.eclipse.che.api.core.model.workspace.WorkspaceStatus;
Expand All @@ -33,7 +32,6 @@

public class WorkspaceStartTrackerMeterBinderTest {

private Random rnd = new Random();
private EventService eventService;
private MeterRegistry registry;
private WorkspaceStartTrackerMeterBinder meterBinder;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.api.metrics;

import static java.util.Arrays.asList;

import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import org.eclipse.che.api.core.model.workspace.WorkspaceStatus;
import org.eclipse.che.api.core.notification.EventService;
import org.eclipse.che.api.workspace.shared.dto.event.WorkspaceStatusEvent;
import org.eclipse.che.dto.server.DtoFactory;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;

public class WorkspaceStopTrackerMeterBinderTest {

private EventService eventService;
private MeterRegistry registry;
private WorkspaceStopTrackerMeterBinder meterBinder;
private Map<String, Long> workspaceStopTime;

@BeforeMethod
public void setUp() {
eventService = new EventService();
registry = new SimpleMeterRegistry();
workspaceStopTime = new ConcurrentHashMap<>();
meterBinder = new WorkspaceStopTrackerMeterBinder(eventService, workspaceStopTime);
meterBinder.bindTo(registry);
}

@Test
public void shouldRecordWorkspaceStopTime() {
// given
// when
eventService.publish(
DtoFactory.newDto(WorkspaceStatusEvent.class)
.withPrevStatus(WorkspaceStatus.RUNNING)
.withStatus(WorkspaceStatus.STOPPING)
.withWorkspaceId("id1"));
// then
Assert.assertTrue(workspaceStopTime.containsKey("id1"));
}

@Test(dataProvider = "allStatusTransitionsWithoutStarting")
public void shouldNotRecordWorkspaceStopTimeForNonStoppingStatuses(
WorkspaceStatus from, WorkspaceStatus to) {
// given
// when
eventService.publish(
DtoFactory.newDto(WorkspaceStatusEvent.class)
.withPrevStatus(from)
.withStatus(to)
.withWorkspaceId("id1"));
// then
Assert.assertTrue(workspaceStopTime.isEmpty());
}

@Test
public void shouldCountSuccessfulStart() {
// given
workspaceStopTime.put("id1", System.currentTimeMillis() - 60 * 1000);
// when
eventService.publish(
DtoFactory.newDto(WorkspaceStatusEvent.class)
.withPrevStatus(WorkspaceStatus.STOPPING)
.withStatus(WorkspaceStatus.STOPPED)
.withWorkspaceId("id1"));

// then

Timer t = registry.find("che.workspace.stop.time").tag("result", "success").timer();
Assert.assertEquals(t.count(), 1);
Assert.assertTrue(t.totalTime(TimeUnit.MILLISECONDS) >= 60 * 1000);
}

@DataProvider
public Object[][] allStatusTransitionsWithoutStarting() {
List<List<WorkspaceStatus>> transitions = new ArrayList<>(9);

for (WorkspaceStatus from : WorkspaceStatus.values()) {
for (WorkspaceStatus to : WorkspaceStatus.values()) {
if ((from == WorkspaceStatus.RUNNING || from == WorkspaceStatus.STARTING)
&& to == WorkspaceStatus.STOPPING) {
continue;
}

transitions.add(asList(from, to));
}
}

return transitions.stream().map(List::toArray).toArray(Object[][]::new);
}
}

0 comments on commit cd891d7

Please sign in to comment.