Skip to content

Commit

Permalink
feat: add prometheus metrics endpoints
Browse files Browse the repository at this point in the history
  • Loading branch information
aaydin-tr committed Jul 24, 2024
1 parent 6be3fa9 commit 01818a3
Show file tree
Hide file tree
Showing 4 changed files with 151 additions and 27 deletions.
11 changes: 10 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,26 @@ require (

require (
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fasthttp/router v1.5.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.19.1 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.48.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/savsgio/gotils v0.0.0-20240704082632-aef3928b8a38 // indirect
github.com/tklauser/go-sysconf v0.3.14 // indirect
github.com/tklauser/numcpus v0.8.0 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fastrand v1.1.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/sys v0.22.0 // indirect
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
)
20 changes: 20 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,22 @@ github.com/aaydin-tr/http2 v0.1.0 h1:+1mEbPCDma8c7BMoXY/GD57KpR55BSBbs7xFSeC/MPM
github.com/aaydin-tr/http2 v0.1.0/go.mod h1:SWWo2Ypjm5gItlaUCJsxlfmnJNpNmhg+tqOZYQxrdmk=
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fasthttp/router v1.5.2 h1:ckJCCdV7hWkkrMeId3WfEhz+4Gyyf6QPwxi/RHIMZ6I=
github.com/fasthttp/router v1.5.2/go.mod h1:C8EY53ozOwpONyevc/V7Gr8pqnEjwnkFFqPo1alAGs0=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
Expand All @@ -19,6 +26,16 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
github.com/savsgio/gotils v0.0.0-20240704082632-aef3928b8a38 h1:D0vL7YNisV2yqE55+q0lFuGse6U8lxlg7fYTctlT5Gc=
github.com/savsgio/gotils v0.0.0-20240704082632-aef3928b8a38/go.mod h1:sM7Mt7uEoCeFSCBM+qBrqvEo+/9vdmj19wzp3yzUhmg=
github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
Expand All @@ -45,8 +62,11 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
58 changes: 32 additions & 26 deletions internal/monitoring/monitoring.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,13 @@ import (
"time"

"github.com/aaydin-tr/divisor/core/types"
"github.com/fasthttp/router"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/shirou/gopsutil/cpu"
"github.com/shirou/gopsutil/mem"
"github.com/shirou/gopsutil/process"
"github.com/valyala/fasthttp"
"github.com/valyala/fasthttp/fasthttpadaptor"
"go.uber.org/zap"
)

Expand Down Expand Up @@ -83,33 +86,36 @@ func getServerStats(server *fasthttp.Server, proxiesStats []types.ProxyStat) Mon
}

func StartMonitoringServer(server *fasthttp.Server, proxies types.IBalancer, addr string) {
r := router.New()
init_prometheus()
go func() {
for {
updatePrometheusMetrics(getServerStats(server, proxies.Stats()))
time.Sleep(5 * time.Second)
}
}()

r.GET("/", func(ctx *fasthttp.RequestCtx) {
ctx.Response.Header.Set("Content-Type", "text/html")
ctx.Response.SetBodyString(index)
})

r.GET("/stats", func(ctx *fasthttp.RequestCtx) {
ctx.Response.Header.Set("Content-Type", "application/json")
m := getServerStats(server, proxies.Stats())
by, err := json.Marshal(m)
if err != nil {
zap.S().Errorf("Error while parsing json, err: %v", err)
return
}

ctx.Response.SetBodyRaw(by)
})

r.GET("/metrics", fasthttpadaptor.NewFastHTTPHandler(promhttp.Handler()))

monitoringServer := fasthttp.Server{
Handler: func(ctx *fasthttp.RequestCtx) {
path, method := string(ctx.Request.URI().Path()), string(ctx.Request.Header.Method())
ctx.Response.Header.Set("Access-Control-Allow-Credentials", "true")
ctx.Response.Header.Set("Access-Control-Allow-Origin", "*")

if path == "/" && method == "GET" {
ctx.Response.Header.Set("Content-Type", "text/html")
ctx.Response.SetBodyString(index)
return
} else if path == "/stats" && method == "GET" {
ctx.Response.Header.Set("Content-Type", "application/json")

m := getServerStats(server, proxies.Stats())
by, err := json.Marshal(m)
if err != nil {
zap.S().Errorf("Error while parsing json, err: %v", err)
return
}

ctx.Response.SetBodyRaw(by)
return
}

ctx.Response.SetStatusCode(fasthttp.StatusNotFound)

},
Handler: r.Handler,
MaxIdleWorkerDuration: 15 * time.Second,
TCPKeepalivePeriod: 15 * time.Second,
TCPKeepalive: true,
Expand Down
89 changes: 89 additions & 0 deletions internal/monitoring/prometheus.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package monitoring

import (
"github.com/prometheus/client_golang/prometheus"
)

var (
processMemoryPercent = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "process_memory_percent",
Help: "Process memory usage percent",
})
totalMemoryPercent = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "total_memory_percent",
Help: "Total memory usage percent",
})
processMemoryMB = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "process_memory_mb",
Help: "Process memory usage in MB",
})
processCPUPercent = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "process_cpu_percent",
Help: "Process CPU usage percent",
})
totalCPUPercent = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "total_cpu_percent",
Help: "Total CPU usage percent",
})
totalGoroutine = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "total_goroutine",
Help: "Total number of goroutines",
})
openConnCount = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "open_conn_count",
Help: "Open connection count",
})

backendTotalReqCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "backend_total_request_count",
Help: "Total request count for each backend",
}, []string{"address"})
backendAvgResTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "backend_average_response_time",
Help: "Average response time for each backend",
}, []string{"address"})
backendConnsCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "backend_connection_count",
Help: "Number of connections for each backend",
}, []string{"address"})
backendAlive = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "backend_alive",
Help: "Whether the backend is alive or not",
}, []string{"address"})
)

func init_prometheus() {
prometheus.MustRegister(processMemoryPercent)
prometheus.MustRegister(totalMemoryPercent)
prometheus.MustRegister(processMemoryMB)
prometheus.MustRegister(processCPUPercent)
prometheus.MustRegister(totalCPUPercent)
prometheus.MustRegister(totalGoroutine)
prometheus.MustRegister(openConnCount)
prometheus.MustRegister(backendTotalReqCount)
prometheus.MustRegister(backendAvgResTime)
prometheus.MustRegister(backendConnsCount)
prometheus.MustRegister(backendAlive)
}

func updatePrometheusMetrics(m Monitoring) {
processMemoryPercent.Set(float64(m.Memory.ProcessPercent))
totalMemoryPercent.Set(float64(m.Memory.TotalPercent))
processMemoryMB.Set(m.Memory.ProcessMB)
processCPUPercent.Set(m.Cpu.ProcessPercent)
totalCPUPercent.Set(m.Cpu.TotalPercent)
totalGoroutine.Set(float64(m.TotalGoroutine))
openConnCount.Set(float64(m.OpenConnectionCount))

for _, backend := range m.Backends {
backendTotalReqCount.WithLabelValues(backend.Addr).Set(float64(backend.TotalReqCount))
backendAvgResTime.WithLabelValues(backend.Addr).Set(backend.AvgResTime)
backendConnsCount.WithLabelValues(backend.Addr).Set(float64(backend.ConnsCount))
backendAlive.WithLabelValues(backend.Addr).Set(func() float64 {
if backend.IsHostAlive {
return 1
}
return 0
}())
}
}

0 comments on commit 01818a3

Please sign in to comment.