diff --git a/go.mod b/go.mod index d2a6318..65f0554 100644 --- a/go.mod +++ b/go.mod @@ -14,11 +14,19 @@ require ( require ( github.com/andybalholm/brotli v1.1.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fasthttp/router v1.5.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.19.1 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/savsgio/gotils v0.0.0-20240704082632-aef3928b8a38 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect @@ -26,5 +34,6 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/sys v0.22.0 // indirect - gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 93e501f..f94c81c 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,14 @@ github.com/aaydin-tr/http2 v0.1.0 h1:+1mEbPCDma8c7BMoXY/GD57KpR55BSBbs7xFSeC/MPM github.com/aaydin-tr/http2 v0.1.0/go.mod h1:SWWo2Ypjm5gItlaUCJsxlfmnJNpNmhg+tqOZYQxrdmk= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fasthttp/router v1.5.2 h1:ckJCCdV7hWkkrMeId3WfEhz+4Gyyf6QPwxi/RHIMZ6I= +github.com/fasthttp/router v1.5.2/go.mod h1:C8EY53ozOwpONyevc/V7Gr8pqnEjwnkFFqPo1alAGs0= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -11,6 +17,7 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -19,6 +26,16 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/savsgio/gotils v0.0.0-20240704082632-aef3928b8a38 h1:D0vL7YNisV2yqE55+q0lFuGse6U8lxlg7fYTctlT5Gc= +github.com/savsgio/gotils v0.0.0-20240704082632-aef3928b8a38/go.mod h1:sM7Mt7uEoCeFSCBM+qBrqvEo+/9vdmj19wzp3yzUhmg= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= @@ -45,8 +62,11 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/monitoring/monitoring.go b/internal/monitoring/monitoring.go index 726e9f9..8cd7de3 100644 --- a/internal/monitoring/monitoring.go +++ b/internal/monitoring/monitoring.go @@ -9,10 +9,13 @@ import ( "time" "github.com/aaydin-tr/divisor/core/types" + "github.com/fasthttp/router" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/shirou/gopsutil/cpu" "github.com/shirou/gopsutil/mem" "github.com/shirou/gopsutil/process" "github.com/valyala/fasthttp" + "github.com/valyala/fasthttp/fasthttpadaptor" "go.uber.org/zap" ) @@ -83,33 +86,36 @@ func getServerStats(server *fasthttp.Server, proxiesStats []types.ProxyStat) Mon } func StartMonitoringServer(server *fasthttp.Server, proxies types.IBalancer, addr string) { + r := router.New() + init_prometheus() + go func() { + for { + updatePrometheusMetrics(getServerStats(server, proxies.Stats())) + time.Sleep(5 * time.Second) + } + }() + + r.GET("/", func(ctx *fasthttp.RequestCtx) { + ctx.Response.Header.Set("Content-Type", "text/html") + ctx.Response.SetBodyString(index) + }) + + r.GET("/stats", func(ctx *fasthttp.RequestCtx) { + ctx.Response.Header.Set("Content-Type", "application/json") + m := getServerStats(server, proxies.Stats()) + by, err := json.Marshal(m) + if err != nil { + zap.S().Errorf("Error while parsing json, err: %v", err) + return + } + + ctx.Response.SetBodyRaw(by) + }) + + r.GET("/metrics", fasthttpadaptor.NewFastHTTPHandler(promhttp.Handler())) + monitoringServer := fasthttp.Server{ - Handler: func(ctx *fasthttp.RequestCtx) { - path, method := string(ctx.Request.URI().Path()), string(ctx.Request.Header.Method()) - ctx.Response.Header.Set("Access-Control-Allow-Credentials", "true") - ctx.Response.Header.Set("Access-Control-Allow-Origin", "*") - - if path == "/" && method == "GET" { - ctx.Response.Header.Set("Content-Type", "text/html") - ctx.Response.SetBodyString(index) - return - } else if path == "/stats" && method == "GET" { - ctx.Response.Header.Set("Content-Type", "application/json") - - m := getServerStats(server, proxies.Stats()) - by, err := json.Marshal(m) - if err != nil { - zap.S().Errorf("Error while parsing json, err: %v", err) - return - } - - ctx.Response.SetBodyRaw(by) - return - } - - ctx.Response.SetStatusCode(fasthttp.StatusNotFound) - - }, + Handler: r.Handler, MaxIdleWorkerDuration: 15 * time.Second, TCPKeepalivePeriod: 15 * time.Second, TCPKeepalive: true, diff --git a/internal/monitoring/prometheus.go b/internal/monitoring/prometheus.go new file mode 100644 index 0000000..c5ad9de --- /dev/null +++ b/internal/monitoring/prometheus.go @@ -0,0 +1,89 @@ +package monitoring + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var ( + processMemoryPercent = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "process_memory_percent", + Help: "Process memory usage percent", + }) + totalMemoryPercent = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "total_memory_percent", + Help: "Total memory usage percent", + }) + processMemoryMB = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "process_memory_mb", + Help: "Process memory usage in MB", + }) + processCPUPercent = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "process_cpu_percent", + Help: "Process CPU usage percent", + }) + totalCPUPercent = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "total_cpu_percent", + Help: "Total CPU usage percent", + }) + totalGoroutine = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "total_goroutine", + Help: "Total number of goroutines", + }) + openConnCount = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "open_conn_count", + Help: "Open connection count", + }) + + backendTotalReqCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "backend_total_request_count", + Help: "Total request count for each backend", + }, []string{"address"}) + backendAvgResTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "backend_average_response_time", + Help: "Average response time for each backend", + }, []string{"address"}) + backendConnsCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "backend_connection_count", + Help: "Number of connections for each backend", + }, []string{"address"}) + backendAlive = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "backend_alive", + Help: "Whether the backend is alive or not", + }, []string{"address"}) +) + +func init_prometheus() { + prometheus.MustRegister(processMemoryPercent) + prometheus.MustRegister(totalMemoryPercent) + prometheus.MustRegister(processMemoryMB) + prometheus.MustRegister(processCPUPercent) + prometheus.MustRegister(totalCPUPercent) + prometheus.MustRegister(totalGoroutine) + prometheus.MustRegister(openConnCount) + prometheus.MustRegister(backendTotalReqCount) + prometheus.MustRegister(backendAvgResTime) + prometheus.MustRegister(backendConnsCount) + prometheus.MustRegister(backendAlive) +} + +func updatePrometheusMetrics(m Monitoring) { + processMemoryPercent.Set(float64(m.Memory.ProcessPercent)) + totalMemoryPercent.Set(float64(m.Memory.TotalPercent)) + processMemoryMB.Set(m.Memory.ProcessMB) + processCPUPercent.Set(m.Cpu.ProcessPercent) + totalCPUPercent.Set(m.Cpu.TotalPercent) + totalGoroutine.Set(float64(m.TotalGoroutine)) + openConnCount.Set(float64(m.OpenConnectionCount)) + + for _, backend := range m.Backends { + backendTotalReqCount.WithLabelValues(backend.Addr).Set(float64(backend.TotalReqCount)) + backendAvgResTime.WithLabelValues(backend.Addr).Set(backend.AvgResTime) + backendConnsCount.WithLabelValues(backend.Addr).Set(float64(backend.ConnsCount)) + backendAlive.WithLabelValues(backend.Addr).Set(func() float64 { + if backend.IsHostAlive { + return 1 + } + return 0 + }()) + } +}