services/horizon: Add additional DB metrics (#2844)

Adds new DB metrics:
* `db_in_use_connections` - number of opened DB connections in use (not
idle),
* `db_wait_count` - number of connections waited for,
* `db_wait_duration` - total time blocked waiting for a new connection.

New metric will help add new alerts connected to DB connection pool.

services/horizon/CHANGELOG.md

@@ -3,6 +3,13 @@
 All notable changes to this project will be documented in this
 file. This project adheres to [Semantic Versioning](http://semver.org/).x
 
+## Unreleased
+
+* Add new DB metrics ([#2844](https://github.com/stellar/go/pull/2844)):
+  * `db_in_use_connections` - number of opened DB connections in use (not idle),
+  * `db_wait_count` - number of connections waited for,
+  * `db_wait_duration` - total time blocked waiting for a new connection.
+
 ## v1.6.0
 
 * Add `--parallel-workers` and `--parallel-job-size` to `horizon db reingest range`. `--parallel-workers` will parallelize reingestion using the supplied number of workers. ([#2724](https://github.com/stellar/go/pull/2724))

services/horizon/internal/app.go

@@ -70,7 +70,10 @@ type App struct {
 	metrics                  metrics.Registry
 	historyLatestLedgerGauge metrics.Gauge
 	historyElderLedgerGauge  metrics.Gauge
-	horizonConnGauge         metrics.Gauge
+	dbOpenConnectionsGauge   metrics.Gauge
+	dbInUseConnectionsGauge  metrics.Gauge
+	dbWaitCountGauge         metrics.Gauge
+	dbWaitDurationTimer      metrics.Timer
 	coreLatestLedgerGauge    metrics.Gauge
 	goroutineGauge           metrics.Gauge
 }
@@ -404,12 +407,16 @@ func (a *App) UpdateStellarCoreInfo() {
 // db connections and ledger state
 func (a *App) UpdateMetrics() {
 	a.goroutineGauge.Update(int64(runtime.NumGoroutine()))
+
 	ls := ledger.CurrentState()
 	a.historyLatestLedgerGauge.Update(int64(ls.HistoryLatest))
 	a.historyElderLedgerGauge.Update(int64(ls.HistoryElder))
 	a.coreLatestLedgerGauge.Update(int64(ls.CoreLatest))
 
-	a.horizonConnGauge.Update(int64(a.historyQ.Session.DB.Stats().OpenConnections))
+	a.dbOpenConnectionsGauge.Update(int64(a.historyQ.Session.DB.Stats().OpenConnections))
+	a.dbInUseConnectionsGauge.Update(int64(a.historyQ.Session.DB.Stats().InUse))
+	a.dbWaitCountGauge.Update(int64(a.historyQ.Session.DB.Stats().WaitCount))
+	a.dbWaitDurationTimer.Update(a.historyQ.Session.DB.Stats().WaitDuration)
 }
 
 // DeleteUnretainedHistory forwards to the app's reaper.  See

services/horizon/internal/init.go

@@ -119,17 +119,26 @@ func initLogglyLog(app *App) {
 }
 
 func initDbMetrics(app *App) {
-	app.historyLatestLedgerGauge = metrics.NewGauge()
-	app.historyElderLedgerGauge = metrics.NewGauge()
-	app.coreLatestLedgerGauge = metrics.NewGauge()
-	app.horizonConnGauge = metrics.NewGauge()
 	app.goroutineGauge = metrics.NewGauge()
+	app.metrics.Register("goroutines", app.goroutineGauge)
+
+	app.historyLatestLedgerGauge = metrics.NewGauge()
 	app.metrics.Register("history.latest_ledger", app.historyLatestLedgerGauge)
+	app.historyElderLedgerGauge = metrics.NewGauge()
 	app.metrics.Register("history.elder_ledger", app.historyElderLedgerGauge)
+	app.coreLatestLedgerGauge = metrics.NewGauge()
 	app.metrics.Register("stellar_core.latest_ledger", app.coreLatestLedgerGauge)
+
+	app.dbOpenConnectionsGauge = metrics.NewGauge()
+	app.metrics.Register("db.open_connections", app.dbOpenConnectionsGauge)
+	app.dbInUseConnectionsGauge = metrics.NewGauge()
+	app.metrics.Register("db.in_use_connections", app.dbInUseConnectionsGauge)
+	app.dbWaitCountGauge = metrics.NewGauge()
+	app.metrics.Register("db.wait_count", app.dbWaitCountGauge)
+	app.dbWaitDurationTimer = metrics.NewTimer()
+	app.metrics.Register("db.wait_duration", app.dbWaitDurationTimer)
+
 	app.metrics.Register("order_book_stream.latest_ledger", app.orderBookStream.LatestLedgerGauge)
-	app.metrics.Register("history.open_connections", app.horizonConnGauge)
-	app.metrics.Register("goroutines", app.goroutineGauge)
 }
 
 // initIngestMetrics registers the metrics for the ingestion into the provided