From 45fb33cd8efa22be4f199f0a83757088a82772ca Mon Sep 17 00:00:00 2001 From: Mykhailo Kuznietsov Date: Thu, 23 May 2019 11:58:24 +0300 Subject: [PATCH 1/3] Add successful stopped workspaces metric Signed-off-by: Mykhailo Kuznietsov --- ...paceSuccessfulStopAttemptsMeterBinder.java | 54 +++++++++++ .../api/metrics/WsMasterMetricsModule.java | 1 + ...SuccessfulStopAttemptsMeterBinderTest.java | 95 +++++++++++++++++++ 3 files changed, 150 insertions(+) create mode 100644 wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceSuccessfulStopAttemptsMeterBinder.java create mode 100644 wsmaster/che-core-api-metrics/src/test/java/org/eclipse/che/api/metrics/WorkspaceSuccessfulStopAttemptsMeterBinderTest.java diff --git a/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceSuccessfulStopAttemptsMeterBinder.java b/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceSuccessfulStopAttemptsMeterBinder.java new file mode 100644 index 00000000000..52f4b0c9024 --- /dev/null +++ b/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceSuccessfulStopAttemptsMeterBinder.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.api.metrics; + +import static org.eclipse.che.api.metrics.WorkspaceBinders.workspaceMetric; + +import com.google.inject.Inject; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.binder.MeterBinder; +import javax.inject.Singleton; +import org.eclipse.che.api.core.model.workspace.WorkspaceStatus; +import org.eclipse.che.api.core.notification.EventService; +import org.eclipse.che.api.workspace.shared.dto.event.WorkspaceStatusEvent; + +/** Counts number of successfully stopped workspaces. */ +@Singleton +public class WorkspaceSuccessfulStopAttemptsMeterBinder implements MeterBinder { + private final EventService eventService; + + private Counter stoppedCounter; + + @Inject + public WorkspaceSuccessfulStopAttemptsMeterBinder(EventService eventService) { + this.eventService = eventService; + } + + @Override + public void bindTo(MeterRegistry registry) { + stoppedCounter = + Counter.builder(workspaceMetric("stopped.total")) + .description("The count of stopped workspaces") + .register(registry); + + // only subscribe to the event once we have the counters ready + eventService.subscribe( + event -> { + if (event.getPrevStatus() == WorkspaceStatus.STOPPING + && event.getStatus() == WorkspaceStatus.STOPPED) { + stoppedCounter.increment(); + } + }, + WorkspaceStatusEvent.class); + } +} diff --git a/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WsMasterMetricsModule.java b/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WsMasterMetricsModule.java index 778163761f1..ffd69142410 100644 --- a/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WsMasterMetricsModule.java +++ b/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WsMasterMetricsModule.java @@ -30,6 +30,7 @@ protected void configure() { meterMultibinder.addBinding().to(WorkspaceFailureMeterBinder.class); meterMultibinder.addBinding().to(WorkspaceStartTrackerMeterBinder.class); meterMultibinder.addBinding().to(WorkspaceSuccessfulStartAttemptsMeterBinder.class); + meterMultibinder.addBinding().to(WorkspaceSuccessfulStopAttemptsMeterBinder.class); meterMultibinder.addBinding().to(WorkspaceStartAttemptsMeterBinder.class); meterMultibinder.addBinding().to(UserMeterBinder.class); } diff --git a/wsmaster/che-core-api-metrics/src/test/java/org/eclipse/che/api/metrics/WorkspaceSuccessfulStopAttemptsMeterBinderTest.java b/wsmaster/che-core-api-metrics/src/test/java/org/eclipse/che/api/metrics/WorkspaceSuccessfulStopAttemptsMeterBinderTest.java new file mode 100644 index 00000000000..eaea0d8295c --- /dev/null +++ b/wsmaster/che-core-api-metrics/src/test/java/org/eclipse/che/api/metrics/WorkspaceSuccessfulStopAttemptsMeterBinderTest.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.api.metrics; + +import static java.util.Arrays.asList; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.util.ArrayList; +import java.util.List; +import org.eclipse.che.api.core.model.workspace.WorkspaceStatus; +import org.eclipse.che.api.core.notification.EventService; +import org.eclipse.che.api.workspace.shared.dto.event.WorkspaceStatusEvent; +import org.eclipse.che.dto.server.DtoFactory; +import org.testng.Assert; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class WorkspaceSuccessfulStopAttemptsMeterBinderTest { + + private EventService eventService; + private MeterRegistry registry; + + @BeforeMethod + public void setUp() { + eventService = new EventService(); + registry = new SimpleMeterRegistry(); + } + + @Test(dataProvider = "allStatusTransitionsWithoutStopping") + public void shouldNotCollectEvents(WorkspaceStatus from, WorkspaceStatus to) { + // given + WorkspaceSuccessfulStopAttemptsMeterBinder meterBinder = + new WorkspaceSuccessfulStopAttemptsMeterBinder(eventService); + meterBinder.bindTo(registry); + + // when + + eventService.publish( + DtoFactory.newDto(WorkspaceStatusEvent.class) + .withPrevStatus(from) + .withStatus(to) + .withWorkspaceId("id1")); + + // then + Counter successful = registry.find("che.workspace.stopped.total").counter(); + Assert.assertEquals(successful.count(), 0.0); + } + + @Test + public void shouldCollectOnlyStopped() { + // given + WorkspaceSuccessfulStopAttemptsMeterBinder meterBinder = + new WorkspaceSuccessfulStopAttemptsMeterBinder(eventService); + meterBinder.bindTo(registry); + + // when + eventService.publish( + DtoFactory.newDto(WorkspaceStatusEvent.class) + .withPrevStatus(WorkspaceStatus.STOPPING) + .withStatus(WorkspaceStatus.STOPPED) + .withWorkspaceId("id1")); + // then + Counter successful = registry.find("che.workspace.stopped.total").counter(); + Assert.assertEquals(successful.count(), 1.0); + } + + @DataProvider + public Object[][] allStatusTransitionsWithoutStopping() { + List> transitions = new ArrayList<>(9); + + for (WorkspaceStatus from : WorkspaceStatus.values()) { + for (WorkspaceStatus to : WorkspaceStatus.values()) { + if (from == WorkspaceStatus.STOPPING && to == WorkspaceStatus.STOPPED) { + continue; + } + + transitions.add(asList(from, to)); + } + } + + return transitions.stream().map(List::toArray).toArray(Object[][]::new); + } +} From 4060fed16e9efaef1779b153910e1b8b83043b74 Mon Sep 17 00:00:00 2001 From: Mykhailo Kuznietsov Date: Thu, 23 May 2019 17:15:54 +0300 Subject: [PATCH 2/3] fixup! Add successful stopped workspaces metric Signed-off-by: Mykhailo Kuznietsov --- ...paceSuccessfulStopAttemptsMeterBinder.java | 5 +++-- ...SuccessfulStopAttemptsMeterBinderTest.java | 21 ++++++++++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceSuccessfulStopAttemptsMeterBinder.java b/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceSuccessfulStopAttemptsMeterBinder.java index 52f4b0c9024..af9aa32e6f4 100644 --- a/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceSuccessfulStopAttemptsMeterBinder.java +++ b/wsmaster/che-core-api-metrics/src/main/java/org/eclipse/che/api/metrics/WorkspaceSuccessfulStopAttemptsMeterBinder.java @@ -44,8 +44,9 @@ public void bindTo(MeterRegistry registry) { // only subscribe to the event once we have the counters ready eventService.subscribe( event -> { - if (event.getPrevStatus() == WorkspaceStatus.STOPPING - && event.getStatus() == WorkspaceStatus.STOPPED) { + if ((event.getError() == null) + && (event.getPrevStatus() == WorkspaceStatus.STOPPING + && event.getStatus() == WorkspaceStatus.STOPPED)) { stoppedCounter.increment(); } }, diff --git a/wsmaster/che-core-api-metrics/src/test/java/org/eclipse/che/api/metrics/WorkspaceSuccessfulStopAttemptsMeterBinderTest.java b/wsmaster/che-core-api-metrics/src/test/java/org/eclipse/che/api/metrics/WorkspaceSuccessfulStopAttemptsMeterBinderTest.java index eaea0d8295c..2a9ed200aa3 100644 --- a/wsmaster/che-core-api-metrics/src/test/java/org/eclipse/che/api/metrics/WorkspaceSuccessfulStopAttemptsMeterBinderTest.java +++ b/wsmaster/che-core-api-metrics/src/test/java/org/eclipse/che/api/metrics/WorkspaceSuccessfulStopAttemptsMeterBinderTest.java @@ -59,7 +59,7 @@ public void shouldNotCollectEvents(WorkspaceStatus from, WorkspaceStatus to) { } @Test - public void shouldCollectOnlyStopped() { + public void shouldCollectOnlyStoppedWithoutError() { // given WorkspaceSuccessfulStopAttemptsMeterBinder meterBinder = new WorkspaceSuccessfulStopAttemptsMeterBinder(eventService); @@ -76,6 +76,25 @@ public void shouldCollectOnlyStopped() { Assert.assertEquals(successful.count(), 1.0); } + @Test + public void shouldNotCollectStoppedWithError() { + // given + WorkspaceSuccessfulStopAttemptsMeterBinder meterBinder = + new WorkspaceSuccessfulStopAttemptsMeterBinder(eventService); + meterBinder.bindTo(registry); + + // when + eventService.publish( + DtoFactory.newDto(WorkspaceStatusEvent.class) + .withPrevStatus(WorkspaceStatus.STOPPING) + .withStatus(WorkspaceStatus.STOPPED) + .withError("Error during workspace stop") + .withWorkspaceId("id1")); + // then + Counter successful = registry.find("che.workspace.stopped.total").counter(); + Assert.assertEquals(successful.count(), 0.0); + } + @DataProvider public Object[][] allStatusTransitionsWithoutStopping() { List> transitions = new ArrayList<>(9); From 7281bd2f6181057ee47807c6d90b2d815aea66c8 Mon Sep 17 00:00:00 2001 From: Mykhailo Kuznietsov Date: Fri, 24 May 2019 11:56:37 +0300 Subject: [PATCH 3/3] fixup! fixup! Add successful stopped workspaces metric Signed-off-by: Mykhailo Kuznietsov --- .../openshift/templates/che-monitoring.yaml | 142 ++++++++++++++++-- 1 file changed, 128 insertions(+), 14 deletions(-) diff --git a/deploy/openshift/templates/che-monitoring.yaml b/deploy/openshift/templates/che-monitoring.yaml index 69434b504f7..70ded85c0c3 100644 --- a/deploy/openshift/templates/che-monitoring.yaml +++ b/deploy/openshift/templates/che-monitoring.yaml @@ -3897,6 +3897,7 @@ objects: "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, + "options": {}, "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -3978,6 +3979,7 @@ objects: "maxDataPoints": 100, "nullPointMode": "connected", "nullText": null, + "options": {}, "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -4060,12 +4062,14 @@ objects: "linewidth": 1, "links": [], "nullPointMode": "null as zero", + "options": {}, "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], + "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ @@ -4131,6 +4135,7 @@ objects: "dashLength": 10, "dashes": false, "datasource": "Che", + "description": "", "fill": 1, "gridPos": { "h": 5, @@ -4138,12 +4143,13 @@ objects: "x": 5, "y": 7 }, - "id": 8, + "id": 48, + "interval": "", "legend": { - "avg": false, + "avg": true, "current": true, - "max": false, - "min": false, + "max": true, + "min": true, "show": true, "total": false, "values": true @@ -4151,30 +4157,38 @@ objects: "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "null", + "nullPointMode": "null as zero", + "options": {}, "paceLength": 10, "percentage": false, - "pointradius": 5, + "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "sum(che_workspace_failure_total)", + "expr": "sum(increase(che_workspace_failure_total{while=\"STOPPING\"} [1h])) /(sum(increase(che_workspace_failure_total{while=\"STOPPING\"} [1h])) + sum(increase(che_workspace_stopped_total [1h])))", "format": "time_series", "intervalFactor": 1, - "legendFormat": "failure", + "legendFormat": "fail", "refId": "A" + }, + { + "expr": "sum(increase(che_workspace_stopped_total [1h]))/(sum(increase(che_workspace_failure_total{while=\"STOPPING\"} [1h])) + sum(increase(che_workspace_stopped_total [1h])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "success", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Workspace Failures", + "title": "Workspace stop rate", "tooltip": { "shared": true, "sort": 0, @@ -4190,7 +4204,7 @@ objects: }, "yaxes": [ { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, @@ -4203,7 +4217,7 @@ objects: "logBase": 1, "max": null, "min": null, - "show": true + "show": false } ], "yaxis": { @@ -4220,7 +4234,7 @@ objects: "fill": 1, "gridPos": { "h": 5, - "w": 6, + "w": 5, "x": 11, "y": 7 }, @@ -4238,6 +4252,7 @@ objects: "linewidth": 1, "links": [], "nullPointMode": "null", + "options": {}, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -4307,7 +4322,7 @@ objects: "gridPos": { "h": 5, "w": 7, - "x": 17, + "x": 16, "y": 7 }, "id": 10, @@ -4324,6 +4339,7 @@ objects: "linewidth": 1, "links": [], "nullPointMode": "null", + "options": {}, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -4412,12 +4428,14 @@ objects: "linewidth": 1, "links": [], "nullPointMode": "null as zero", + "options": {}, "paceLength": 10, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], + "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ @@ -4471,6 +4489,93 @@ objects: "alignLevel": null } }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Che", + "fill": 1, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 12 + }, + "id": 8, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(che_workspace_failure_total)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "failure", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Workspace Failures", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "collapsed": false, "gridPos": { @@ -4511,6 +4616,7 @@ objects: "linewidth": 1, "links": [], "nullPointMode": "null", + "options": {}, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -4612,6 +4718,7 @@ objects: "linewidth": 1, "links": [], "nullPointMode": "null as zero", + "options": {}, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -4724,6 +4831,7 @@ objects: "linewidth": 1, "links": [], "nullPointMode": "null as zero", + "options": {}, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -4823,6 +4931,7 @@ objects: "linewidth": 1, "links": [], "nullPointMode": "null", + "options": {}, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -4919,6 +5028,7 @@ objects: "linewidth": 1, "links": [], "nullPointMode": "null as zero", + "options": {}, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -5012,6 +5122,7 @@ objects: "linewidth": 1, "links": [], "nullPointMode": "null as zero", + "options": {}, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -5107,6 +5218,7 @@ objects: "linewidth": 1, "links": [], "nullPointMode": "null as zero", + "options": {}, "paceLength": 10, "percentage": false, "pointradius": 5, @@ -5215,6 +5327,7 @@ objects: "show": true }, "links": [], + "options": {}, "targets": [ { "expr": "increase(che_workspace_start_time_seconds_bucket{result=\"fail\"}[1h])", @@ -5278,6 +5391,7 @@ objects: "show": true }, "links": [], + "options": {}, "targets": [ { "expr": "increase(che_workspace_start_time_seconds_bucket{result=\"success\"} [1h])",