Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GH-2277 Fix failure when StatisticsHandler returns NaN #2278

Merged
merged 2 commits into from
Nov 27, 2024
Merged

GH-2277 Fix failure when StatisticsHandler returns NaN #2278

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5e62139
Fixes #2277
solonovamax Nov 11, 2024
4fc658f
Add unit tests for metrics
solonovamax Nov 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 67 additions & 56 deletions ...rometheus-plugin/src/main/kotlin/com/reposilite/plugin/prometheus/metrics/JettyMetrics.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package com.reposilite.plugin.prometheus.metrics

import io.prometheus.metrics.core.metrics.Counter
import io.prometheus.metrics.core.metrics.CounterWithCallback
import io.prometheus.metrics.core.metrics.GaugeWithCallback
import io.prometheus.metrics.core.metrics.Summary
import io.prometheus.metrics.model.registry.PrometheusRegistry
import org.eclipse.jetty.io.Connection
import org.eclipse.jetty.server.handler.StatisticsHandler
import kotlin.time.Duration.Companion.milliseconds
Expand All @@ -12,155 +12,141 @@ import io.prometheus.metrics.model.snapshots.Unit as MetricsUnit


object JettyMetrics : Connection.Listener {
val responseTimeSummary: Summary = Summary.builder()
.name("jetty_response_time_seconds")
.help("Time spent for a response")
.labelNames("code")
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register()

val requestBytesSummary: Summary = Summary.builder()
.name("jetty_request_bytes")
.help("Size in bytes of incoming requests")
.unit(MetricsUnit.BYTES)
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register()

val responseBytesSummary: Summary = Summary.builder()
.name("jetty_response_bytes")
.help("Size in bytes of outgoing responses")
.unit(MetricsUnit.BYTES)
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register()

fun register(statisticsHandler: StatisticsHandler) {
lateinit var responseTimeSummary: Summary
private set

private lateinit var requestBytesSummary: Summary

private lateinit var responseBytesSummary: Summary

fun register(statisticsHandler: StatisticsHandler, registry: PrometheusRegistry = PrometheusRegistry.defaultRegistry) {
CounterWithCallback.builder()
.name("jetty_requests_total")
.help("Number of requests")
.callback { callback -> callback.call(statisticsHandler.requests.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_requests_active")
.help("Number of requests currently active")
.callback { callback -> callback.call(statisticsHandler.requestsActive.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_requests_active_max")
.help("Maximum number of requests that have been active at once")
.callback { callback -> callback.call(statisticsHandler.requestsActiveMax.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_request_time_seconds_max")
.help("Maximum time spent handling requests")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statisticsHandler.requestTimeMax.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.callback { callback -> callback.call(statisticsHandler.requestTimeMax.toMillisFinite()) }
.register(registry)

CounterWithCallback.builder()
.name("jetty_request_time_seconds_total")
.help("Total time spent in all request handling")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statisticsHandler.requestTimeTotal.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.callback { callback -> callback.call(statisticsHandler.requestTimeTotal.toMillisFinite()) }
.register(registry)

GaugeWithCallback.builder()
.name("jetty_request_time_seconds_mean")
.help("Mean time spent in all request handling")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statisticsHandler.requestTimeMean.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.callback { callback -> callback.call(statisticsHandler.requestTimeMean.toMillisFinite()) }
.register(registry)

GaugeWithCallback.builder()
.name("jetty_request_time_seconds_stddev")
.help("Standard deviation of time spent in all request handling")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statisticsHandler.requestTimeStdDev.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.callback { callback -> callback.call(statisticsHandler.requestTimeStdDev.toMillisFinite()) }
.register(registry)

CounterWithCallback.builder()
.name("jetty_dispatched_total")
.help("Number of dispatches")
.callback { callback -> callback.call(statisticsHandler.dispatched.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_dispatched_active")
.help("Number of dispatches currently active")
.callback { callback -> callback.call(statisticsHandler.dispatchedActive.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_dispatched_active_max")
.help("Maximum number of active dispatches being handled")
.callback { callback -> callback.call(statisticsHandler.dispatchedActiveMax.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_dispatched_time_seconds_max")
.help("Maximum time spent in dispatch handling")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statisticsHandler.dispatchedTimeMax.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.callback { callback -> callback.call(statisticsHandler.dispatchedTimeMax.toMillisFinite()) }
.register(registry)

CounterWithCallback.builder()
.name("jetty_dispatched_time_seconds_total")
.help("Total time spent in dispatch handling")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statisticsHandler.dispatchedTimeTotal.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.callback { callback -> callback.call(statisticsHandler.dispatchedTimeTotal.toMillisFinite()) }
.register(registry)

GaugeWithCallback.builder()
.name("jetty_dispatched_time_seconds_mean")
.help("Mean time spent in dispatch handling")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statisticsHandler.dispatchedTimeMean.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.callback { callback -> callback.call(statisticsHandler.dispatchedTimeMean.toMillisFinite()) }
.register(registry)

GaugeWithCallback.builder()
.name("jetty_dispatched_time_seconds_stddev")
.help("Standard deviation of time spent in dispatch handling")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statisticsHandler.dispatchedTimeStdDev.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.callback { callback -> callback.call(statisticsHandler.dispatchedTimeStdDev.toMillisFinite()) }
.register(registry)

CounterWithCallback.builder()
.name("jetty_async_requests_total")
.help("Total number of async requests")
.callback { callback -> callback.call(statisticsHandler.asyncRequests.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_async_requests_waiting")
.help("Currently waiting async requests")
.callback { callback -> callback.call(statisticsHandler.asyncRequestsWaiting.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_async_requests_waiting_max")
.help("Maximum number of waiting async requests")
.callback { callback -> callback.call(statisticsHandler.asyncRequestsWaitingMax.toDouble()) }
.register()
.register(registry)

CounterWithCallback.builder()
.name("jetty_async_dispatches_total")
.help("Number of requested that have been asynchronously dispatched")
.callback { callback -> callback.call(statisticsHandler.asyncDispatches.toDouble()) }
.register()
.register(registry)

CounterWithCallback.builder()
.name("jetty_expires_total")
.help("Number of async requests requests that have expired")
.callback { callback -> callback.call(statisticsHandler.expires.toDouble()) }
.register()
.register(registry)

CounterWithCallback.builder()
.name("jetty_errors_total")
.help("Number of async errors that occurred")
.callback { callback -> callback.call(statisticsHandler.errors.toDouble()) }
.register()
.register(registry)

CounterWithCallback.builder()
.name("jetty_responses_total")
Expand All @@ -173,26 +159,47 @@ object JettyMetrics : Connection.Listener {
callback.call(statisticsHandler.responses4xx.toDouble(), "4xx")
callback.call(statisticsHandler.responses5xx.toDouble(), "5xx")
}
.register()
.register(registry)

CounterWithCallback.builder()
.name("jetty_responses_thrown_total")
.help("Number of requests that threw an exception")
.callback { callback -> callback.call(statisticsHandler.responsesThrown.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_stats_seconds")
.help("Time in seconds stats have been collected for")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statisticsHandler.statsOnMs.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.callback { callback -> callback.call(statisticsHandler.statsOnMs.toMillisFinite()) }
.register(registry)

CounterWithCallback.builder()
.name("jetty_responses_bytes_total")
.help("Total number of bytes across all responses")
.callback { callback -> callback.call(statisticsHandler.responsesBytesTotal.toDouble()) }
.register()
.register(registry)

responseTimeSummary = Summary.builder()
.name("jetty_response_time_seconds")
.help("Time spent for a response")
.labelNames("code")
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register(registry)

requestBytesSummary = Summary.builder()
.name("jetty_request_bytes")
.help("Size in bytes of incoming requests")
.unit(MetricsUnit.BYTES)
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register(registry)

responseBytesSummary = Summary.builder()
.name("jetty_response_bytes")
.help("Size in bytes of outgoing responses")
.unit(MetricsUnit.BYTES)
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register(registry)
}

override fun onOpened(connection: Connection) {}
Expand All @@ -201,4 +208,8 @@ object JettyMetrics : Connection.Listener {
requestBytesSummary.observe(connection.bytesIn.toDouble())
responseBytesSummary.observe(connection.bytesOut.toDouble())
}

private fun Double.toMillisFinite() = if (this.isFinite()) this.milliseconds.toDouble(DurationUnit.SECONDS) else this

private fun Long.toMillisFinite() = this.milliseconds.toDouble(DurationUnit.SECONDS)
}
11 changes: 6 additions & 5 deletions ...lugin/src/main/kotlin/com/reposilite/plugin/prometheus/metrics/QueuedThreadPoolMetrics.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
package com.reposilite.plugin.prometheus.metrics

import io.prometheus.metrics.core.metrics.GaugeWithCallback
import io.prometheus.metrics.model.registry.PrometheusRegistry
import org.eclipse.jetty.util.thread.QueuedThreadPool
import io.prometheus.metrics.model.snapshots.Unit as MetricsUnit


object QueuedThreadPoolMetrics {
fun register(queuedThreadPool: QueuedThreadPool) {
fun register(queuedThreadPool: QueuedThreadPool, registry: PrometheusRegistry = PrometheusRegistry.defaultRegistry) {
GaugeWithCallback.builder()
.name("jetty_queued_thread_pool_threads_state")
.help("Number of threads by state")
Expand All @@ -21,25 +22,25 @@ object QueuedThreadPoolMetrics {
callback.call(queuedThreadPool.utilizedThreads.toDouble(), "utilized")
callback.call(queuedThreadPool.maxAvailableThreads.toDouble(), "max_available")
}
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_queued_thread_pool_utilization")
.help("Percentage of threads in use")
.unit(MetricsUnit.RATIO)
.callback { callback -> callback.call(queuedThreadPool.utilizationRate) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_queued_thread_pool_jobs")
.help("Number of total jobs")
.callback { callback -> callback.call(queuedThreadPool.queueSize.toDouble()) }
.register()
.register(registry)

GaugeWithCallback.builder()
.name("jetty_queued_thread_pool_low_on_threads")
.help("Number of total jobs")
.callback { callback -> callback.call(if (queuedThreadPool.isLowOnThreads) 1.0 else 0.0) }
.register()
.register(registry)
}
}
63 changes: 38 additions & 25 deletions ...heus-plugin/src/main/kotlin/com/reposilite/plugin/prometheus/metrics/ReposiliteMetrics.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,48 +6,61 @@ import io.prometheus.metrics.core.metrics.Counter
import io.prometheus.metrics.core.metrics.CounterWithCallback
import io.prometheus.metrics.core.metrics.GaugeWithCallback
import io.prometheus.metrics.core.metrics.Summary
import io.prometheus.metrics.model.registry.PrometheusRegistry
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.DurationUnit
import io.prometheus.metrics.model.snapshots.Unit as MetricsUnit

object ReposiliteMetrics {
// TODO: Remove this? See #2251
val responseFileSizeSummary: Summary = Summary.builder()
.name("reposilite_response_file_size_bytes")
.help("Size in bytes of response files")
.unit(MetricsUnit.BYTES)
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register()
lateinit var responseFileSizeSummary: Summary

// TODO: Remove this? See #2251
val resolvedFileCounter: Counter = Counter.builder()
.name("reposilite_resolved_total")
.help("Total resolved files count")
.register()

val mavenDeployCounter: Counter = Counter.builder()
.name("reposilite_deploy_total")
.help("Total successful deployments count")
.register()

val responseCounter: Counter = Counter.builder()
.name("reposilite_responses_total")
.help("Total response count, filtered to exclude /metrics")
.labelNames("code")
.register()

fun register(statusFacade: StatusFacade, failureFacade: FailureFacade) {
lateinit var resolvedFileCounter: Counter

lateinit var mavenDeployCounter: Counter

lateinit var responseCounter: Counter

fun register(
statusFacade: StatusFacade,
failureFacade: FailureFacade,
registry: PrometheusRegistry = PrometheusRegistry.defaultRegistry,
) {
GaugeWithCallback.builder()
.name("reposilite_uptime_seconds")
.help("Uptime of reposilite")
.unit(MetricsUnit.SECONDS)
.callback { callback -> callback.call(statusFacade.fetchInstanceStatus().uptime.milliseconds.toDouble(DurationUnit.SECONDS)) }
.register()
.register(registry)

CounterWithCallback.builder()
.name("reposilite_failures_total")
.help("Number of failures reposilite has encountered")
.callback { callback -> callback.call(failureFacade.getFailures().size.toDouble()) }
.register()
.register(registry)

responseFileSizeSummary = Summary.builder()
.name("reposilite_response_file_size_bytes")
.help("Size in bytes of response files")
.unit(MetricsUnit.BYTES)
.quantile(0.01, 0.05, 0.1, 0.5, 0.9, 0.95, 0.99)
.register(registry)

resolvedFileCounter = Counter.builder()
.name("reposilite_resolved_total")
.help("Total resolved files count")
.register(registry)

mavenDeployCounter = Counter.builder()
.name("reposilite_deploy_total")
.help("Total successful deployments count")
.register(registry)

responseCounter = Counter.builder()
.name("reposilite_responses_total")
.help("Total response count, filtered to exclude /metrics")
.labelNames("code")
.register(registry)
}
}
Loading
Loading