From f1c84ae1be88713d958157a032eacff810a1d388 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Fri, 8 May 2020 09:53:51 +0200 Subject: [PATCH 01/25] metrics --- cmd/revad/runtime/loader.go | 1 + examples/metrics/metrics.toml | 4 ++++ pkg/metrics/metrics.go | 26 ++++++++++++++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 examples/metrics/metrics.toml create mode 100644 pkg/metrics/metrics.go diff --git a/cmd/revad/runtime/loader.go b/cmd/revad/runtime/loader.go index cae629b1ba..2298fe219b 100644 --- a/cmd/revad/runtime/loader.go +++ b/cmd/revad/runtime/loader.go @@ -30,6 +30,7 @@ import ( _ "github.com/cs3org/reva/pkg/auth/manager/loader" _ "github.com/cs3org/reva/pkg/auth/registry/loader" _ "github.com/cs3org/reva/pkg/meshdirectory/manager/loader" + _ "github.com/cs3org/reva/pkg/metrics" _ "github.com/cs3org/reva/pkg/ocm/invite/manager/loader" _ "github.com/cs3org/reva/pkg/ocm/provider/authorizer/loader" _ "github.com/cs3org/reva/pkg/ocm/share/manager/loader" diff --git a/examples/metrics/metrics.toml b/examples/metrics/metrics.toml new file mode 100644 index 0000000000..f06751e8be --- /dev/null +++ b/examples/metrics/metrics.toml @@ -0,0 +1,4 @@ +[shared] +jwt_secret = "mysecret" + +[http.services.prometheus] diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 0000000000..da807d82d6 --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,26 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package metrics + +func init() { + GetNumUsers() + // import prometheus and define metrics + // define your variables, like active users + // for the mockup: every 2 seconds, increase number of users. +} From 629c46d3ca3214c93d30da319494dc60ba351b96 Mon Sep 17 00:00:00 2001 From: Hugo Gonzalez Labrador Date: Fri, 8 May 2020 09:55:51 +0200 Subject: [PATCH 02/25] metrcs --- pkg/metrics/metrics.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index da807d82d6..8636cf67c3 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -19,7 +19,6 @@ package metrics func init() { - GetNumUsers() // import prometheus and define metrics // define your variables, like active users // for the mockup: every 2 seconds, increase number of users. From 69f656eaa6f33faab65727f0b684866523111d37 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 12 May 2020 11:52:13 +0200 Subject: [PATCH 03/25] Attempt to add metrics to existing collector. --- internal/http/services/prometheus/prometheus.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/http/services/prometheus/prometheus.go b/internal/http/services/prometheus/prometheus.go index 456c8163fd..9d995c20b6 100644 --- a/internal/http/services/prometheus/prometheus.go +++ b/internal/http/services/prometheus/prometheus.go @@ -20,6 +20,7 @@ package prometheus import ( "net/http" + mtrcs "reva/pkg/metrics" "contrib.go.opencensus.io/exporter/prometheus" "github.com/cs3org/reva/pkg/rhttp/global" @@ -31,6 +32,8 @@ import ( func init() { global.Register("prometheus", New) + m := mtrcs.New() + m.SayHello() } // New returns a new prometheus service From bafa6bdb0046ebd64cfd39512b254cb1ffa38929 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 12 May 2020 11:53:20 +0200 Subject: [PATCH 04/25] New metrics pkg. --- pkg/metrics/metrics.go | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 8636cf67c3..0d77955f10 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -18,8 +18,47 @@ package metrics +import ( + "fmt" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + func init() { // import prometheus and define metrics // define your variables, like active users // for the mockup: every 2 seconds, increase number of users. + recordMetrics() +} + +// Metrics metrics struct +type Metrics struct { +} + +// SayHello says hello +func (m *Metrics) SayHello() { + fmt.Printf("Hello from metrics pkg.\n") } + +// New returns a new Metrics +func New() *Metrics { + return &Metrics{} +} + +func recordMetrics() { + go func() { + for { + opsProcessed.Inc() + time.Sleep(2 * time.Second) + } + }() +} + +var ( + opsProcessed = promauto.NewCounter(prometheus.CounterOpts{ + Name: "myapp_processed_ops_total", + Help: "The total number of processed events", + }) +) From 2f0735522424f861e1568860f5bd729007202fa2 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 12 May 2020 11:57:26 +0200 Subject: [PATCH 05/25] Metrics service conf. --- examples/oc-phoenix/metrics.toml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 examples/oc-phoenix/metrics.toml diff --git a/examples/oc-phoenix/metrics.toml b/examples/oc-phoenix/metrics.toml new file mode 100644 index 0000000000..6caa3cc2ae --- /dev/null +++ b/examples/oc-phoenix/metrics.toml @@ -0,0 +1,7 @@ +[shared] +jwt_secret = "Pive-Fumkiu4" + +[http.services.prometheus] + +[http] +address = "localhost:5550" \ No newline at end of file From 93f4fc66016e3364f40f0b22723e32280a18ec36 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 12 May 2020 13:54:42 +0200 Subject: [PATCH 06/25] getNumUsers() stub implemented. --- pkg/metrics/metrics.go | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 0d77955f10..b89fb7491b 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -27,27 +27,11 @@ import ( ) func init() { - // import prometheus and define metrics - // define your variables, like active users - // for the mockup: every 2 seconds, increase number of users. - recordMetrics() + fmt.Printf("init metrics\n") + getNumUsers() } -// Metrics metrics struct -type Metrics struct { -} - -// SayHello says hello -func (m *Metrics) SayHello() { - fmt.Printf("Hello from metrics pkg.\n") -} - -// New returns a new Metrics -func New() *Metrics { - return &Metrics{} -} - -func recordMetrics() { +func getNumUsers() { go func() { for { opsProcessed.Inc() @@ -58,7 +42,7 @@ func recordMetrics() { var ( opsProcessed = promauto.NewCounter(prometheus.CounterOpts{ - Name: "myapp_processed_ops_total", - Help: "The total number of processed events", + Name: "cs3_org_sciencemesh_site_total_num_users", + Help: "The total number of users within this site", }) ) From c1caddedfbb470c70a542a1b145e2e28318ee998 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 12 May 2020 13:56:04 +0200 Subject: [PATCH 07/25] Remove metrics registration. --- .../http/services/prometheus/prometheus.go | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/internal/http/services/prometheus/prometheus.go b/internal/http/services/prometheus/prometheus.go index 9d995c20b6..e2e501784e 100644 --- a/internal/http/services/prometheus/prometheus.go +++ b/internal/http/services/prometheus/prometheus.go @@ -20,20 +20,14 @@ package prometheus import ( "net/http" - mtrcs "reva/pkg/metrics" - "contrib.go.opencensus.io/exporter/prometheus" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" - "github.com/rs/zerolog" - "go.opencensus.io/stats/view" + "github.com/prometheus/client_golang/prometheus/promhttp" ) func init() { global.Register("prometheus", New) - m := mtrcs.New() - m.SayHello() } // New returns a new prometheus service @@ -47,16 +41,18 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) conf.Prefix = "metrics" } - pe, err := prometheus.NewExporter(prometheus.Options{ - Namespace: "revad", - }) - if err != nil { - return nil, errors.Wrap(err, "prometheus: error creating exporter") - } + // pe, err := prometheus.NewExporter(prometheus.Options{ + // Namespace: "revad", + // }) + // if err != nil { + // return nil, errors.Wrap(err, "prometheus: error creating exporter") + // } + + // view.RegisterExporter(pe) - view.RegisterExporter(pe) + // return &svc{prefix: conf.Prefix, h: pe}, nil - return &svc{prefix: conf.Prefix, h: pe}, nil + return &svc{prefix: conf.Prefix, h: promhttp.Handler()}, nil } type config struct { From cd2c6e1ab60ace1a267d9858317d52b8998931c3 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 12 May 2020 16:37:57 +0200 Subject: [PATCH 08/25] A second metric added. --- pkg/metrics/metrics.go | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index b89fb7491b..4a4e7fa411 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -20,6 +20,7 @@ package metrics import ( "fmt" + "math/rand" "time" "github.com/prometheus/client_golang/prometheus" @@ -27,22 +28,45 @@ import ( ) func init() { - fmt.Printf("init metrics\n") + fmt.Printf("Init metrics\n") getNumUsers() + getNumGroups() } func getNumUsers() { + // here we must request the actual number of users from the site + // for now this sets a random dummy value + rand.Seed(time.Now().UnixNano()) go func() { for { - opsProcessed.Inc() + numUsersGauge.Set(float64(rand.Intn(10))) time.Sleep(2 * time.Second) } }() } var ( - opsProcessed = promauto.NewCounter(prometheus.CounterOpts{ + numUsersGauge = promauto.NewGauge(prometheus.GaugeOpts{ Name: "cs3_org_sciencemesh_site_total_num_users", Help: "The total number of users within this site", }) ) + +func getNumGroups() { + // here we must request the actual number of groups from the site + // for now this sets a random dummy value + rand.Seed(time.Now().UnixNano()) + go func() { + for { + numGroupsGauge.Set(float64(rand.Intn(10))) + time.Sleep(2 * time.Second) + } + }() +} + +var ( + numGroupsGauge = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "cs3_org_sciencemesh_site_total_num_groups", + Help: "The total number of groups within this site", + }) +) From 242be2b8290a9f5b6283831d83904fafeae2d3e9 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Wed, 13 May 2020 10:37:36 +0200 Subject: [PATCH 09/25] 3 metrics registered. --- pkg/metrics/metrics.go | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 4a4e7fa411..a53c38c034 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -18,8 +18,9 @@ package metrics +// This package auto registers site metrics in prometheus + import ( - "fmt" "math/rand" "time" @@ -28,25 +29,25 @@ import ( ) func init() { - fmt.Printf("Init metrics\n") + // trigger the actual metric provider functions getNumUsers() getNumGroups() + getStorageUsed() } func getNumUsers() { // here we must request the actual number of users from the site - // for now this sets a random dummy value - rand.Seed(time.Now().UnixNano()) + // for now this is a mockup: a number increasing over time go func() { for { - numUsersGauge.Set(float64(rand.Intn(10))) + numUsersCounter.Add(float64(123)) time.Sleep(2 * time.Second) } }() } var ( - numUsersGauge = promauto.NewGauge(prometheus.GaugeOpts{ + numUsersCounter = promauto.NewCounter(prometheus.CounterOpts{ Name: "cs3_org_sciencemesh_site_total_num_users", Help: "The total number of users within this site", }) @@ -54,11 +55,11 @@ var ( func getNumGroups() { // here we must request the actual number of groups from the site - // for now this sets a random dummy value + // for now this is a mockup: a random number changing over time rand.Seed(time.Now().UnixNano()) go func() { for { - numGroupsGauge.Set(float64(rand.Intn(10))) + numGroupsGauge.Set(float64(rand.Intn(100))) time.Sleep(2 * time.Second) } }() @@ -70,3 +71,21 @@ var ( Help: "The total number of groups within this site", }) ) + +func getStorageUsed() { + // here we must request the actual amount of storage used within the site + // for now this is a mockup: a number increasing over time + go func() { + for { + amountStorageUsed.Add(float64(12345)) + time.Sleep(2 * time.Second) + } + }() +} + +var ( + amountStorageUsed = promauto.NewCounter(prometheus.CounterOpts{ + Name: "cs3_org_sciencemesh_site_amount_storage_used", + Help: "The total amount of storage used within this site", + }) +) From 533d2b18b3d355b47988e621f4b0be00aa6fa40e Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 12 May 2020 11:52:13 +0200 Subject: [PATCH 10/25] Attempt to add metrics to existing collector. --- internal/http/services/prometheus/prometheus.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/http/services/prometheus/prometheus.go b/internal/http/services/prometheus/prometheus.go index e2e501784e..cafa03d131 100644 --- a/internal/http/services/prometheus/prometheus.go +++ b/internal/http/services/prometheus/prometheus.go @@ -20,6 +20,7 @@ package prometheus import ( "net/http" + mtrcs "reva/pkg/metrics" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/mitchellh/mapstructure" @@ -28,6 +29,8 @@ import ( func init() { global.Register("prometheus", New) + m := mtrcs.New() + m.SayHello() } // New returns a new prometheus service From 7152d622986d3d5eeeca636d5524b02471f9b154 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 12 May 2020 11:53:20 +0200 Subject: [PATCH 11/25] New metrics pkg. --- pkg/metrics/metrics.go | 65 ++++++++++++------------------------------ 1 file changed, 19 insertions(+), 46 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index a53c38c034..0d77955f10 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -18,10 +18,8 @@ package metrics -// This package auto registers site metrics in prometheus - import ( - "math/rand" + "fmt" "time" "github.com/prometheus/client_golang/prometheus" @@ -29,63 +27,38 @@ import ( ) func init() { - // trigger the actual metric provider functions - getNumUsers() - getNumGroups() - getStorageUsed() + // import prometheus and define metrics + // define your variables, like active users + // for the mockup: every 2 seconds, increase number of users. + recordMetrics() } -func getNumUsers() { - // here we must request the actual number of users from the site - // for now this is a mockup: a number increasing over time - go func() { - for { - numUsersCounter.Add(float64(123)) - time.Sleep(2 * time.Second) - } - }() +// Metrics metrics struct +type Metrics struct { } -var ( - numUsersCounter = promauto.NewCounter(prometheus.CounterOpts{ - Name: "cs3_org_sciencemesh_site_total_num_users", - Help: "The total number of users within this site", - }) -) - -func getNumGroups() { - // here we must request the actual number of groups from the site - // for now this is a mockup: a random number changing over time - rand.Seed(time.Now().UnixNano()) - go func() { - for { - numGroupsGauge.Set(float64(rand.Intn(100))) - time.Sleep(2 * time.Second) - } - }() +// SayHello says hello +func (m *Metrics) SayHello() { + fmt.Printf("Hello from metrics pkg.\n") } -var ( - numGroupsGauge = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "cs3_org_sciencemesh_site_total_num_groups", - Help: "The total number of groups within this site", - }) -) +// New returns a new Metrics +func New() *Metrics { + return &Metrics{} +} -func getStorageUsed() { - // here we must request the actual amount of storage used within the site - // for now this is a mockup: a number increasing over time +func recordMetrics() { go func() { for { - amountStorageUsed.Add(float64(12345)) + opsProcessed.Inc() time.Sleep(2 * time.Second) } }() } var ( - amountStorageUsed = promauto.NewCounter(prometheus.CounterOpts{ - Name: "cs3_org_sciencemesh_site_amount_storage_used", - Help: "The total amount of storage used within this site", + opsProcessed = promauto.NewCounter(prometheus.CounterOpts{ + Name: "myapp_processed_ops_total", + Help: "The total number of processed events", }) ) From 365ed968502722de0c8edb7de1de286024b905da Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 12 May 2020 13:54:42 +0200 Subject: [PATCH 12/25] getNumUsers() stub implemented. --- pkg/metrics/metrics.go | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 0d77955f10..b89fb7491b 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -27,27 +27,11 @@ import ( ) func init() { - // import prometheus and define metrics - // define your variables, like active users - // for the mockup: every 2 seconds, increase number of users. - recordMetrics() + fmt.Printf("init metrics\n") + getNumUsers() } -// Metrics metrics struct -type Metrics struct { -} - -// SayHello says hello -func (m *Metrics) SayHello() { - fmt.Printf("Hello from metrics pkg.\n") -} - -// New returns a new Metrics -func New() *Metrics { - return &Metrics{} -} - -func recordMetrics() { +func getNumUsers() { go func() { for { opsProcessed.Inc() @@ -58,7 +42,7 @@ func recordMetrics() { var ( opsProcessed = promauto.NewCounter(prometheus.CounterOpts{ - Name: "myapp_processed_ops_total", - Help: "The total number of processed events", + Name: "cs3_org_sciencemesh_site_total_num_users", + Help: "The total number of users within this site", }) ) From fcfbe05c4951ce89a65ce777a029a6338ec765bc Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 12 May 2020 13:56:04 +0200 Subject: [PATCH 13/25] Remove metrics registration. --- internal/http/services/prometheus/prometheus.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/http/services/prometheus/prometheus.go b/internal/http/services/prometheus/prometheus.go index cafa03d131..e2e501784e 100644 --- a/internal/http/services/prometheus/prometheus.go +++ b/internal/http/services/prometheus/prometheus.go @@ -20,7 +20,6 @@ package prometheus import ( "net/http" - mtrcs "reva/pkg/metrics" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/mitchellh/mapstructure" @@ -29,8 +28,6 @@ import ( func init() { global.Register("prometheus", New) - m := mtrcs.New() - m.SayHello() } // New returns a new prometheus service From 8f45f0ea0f066bb1478037b34da8f82eb5abee80 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Tue, 12 May 2020 16:37:57 +0200 Subject: [PATCH 14/25] A second metric added. --- pkg/metrics/metrics.go | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index b89fb7491b..4a4e7fa411 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -20,6 +20,7 @@ package metrics import ( "fmt" + "math/rand" "time" "github.com/prometheus/client_golang/prometheus" @@ -27,22 +28,45 @@ import ( ) func init() { - fmt.Printf("init metrics\n") + fmt.Printf("Init metrics\n") getNumUsers() + getNumGroups() } func getNumUsers() { + // here we must request the actual number of users from the site + // for now this sets a random dummy value + rand.Seed(time.Now().UnixNano()) go func() { for { - opsProcessed.Inc() + numUsersGauge.Set(float64(rand.Intn(10))) time.Sleep(2 * time.Second) } }() } var ( - opsProcessed = promauto.NewCounter(prometheus.CounterOpts{ + numUsersGauge = promauto.NewGauge(prometheus.GaugeOpts{ Name: "cs3_org_sciencemesh_site_total_num_users", Help: "The total number of users within this site", }) ) + +func getNumGroups() { + // here we must request the actual number of groups from the site + // for now this sets a random dummy value + rand.Seed(time.Now().UnixNano()) + go func() { + for { + numGroupsGauge.Set(float64(rand.Intn(10))) + time.Sleep(2 * time.Second) + } + }() +} + +var ( + numGroupsGauge = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "cs3_org_sciencemesh_site_total_num_groups", + Help: "The total number of groups within this site", + }) +) From 0826f72f3d574f576db328825a8d31faeae2692f Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Wed, 13 May 2020 12:36:28 +0200 Subject: [PATCH 15/25] Reset code --- .../http/services/prometheus/prometheus.go | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/http/services/prometheus/prometheus.go b/internal/http/services/prometheus/prometheus.go index e2e501784e..4462e193c8 100644 --- a/internal/http/services/prometheus/prometheus.go +++ b/internal/http/services/prometheus/prometheus.go @@ -21,9 +21,11 @@ package prometheus import ( "net/http" + "contrib.go.opencensus.io/exporter/prometheus" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/mitchellh/mapstructure" - "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/pkg/errors" + "go.opencensus.io/stats/view" ) func init() { @@ -41,18 +43,16 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) conf.Prefix = "metrics" } - // pe, err := prometheus.NewExporter(prometheus.Options{ - // Namespace: "revad", - // }) - // if err != nil { - // return nil, errors.Wrap(err, "prometheus: error creating exporter") - // } - - // view.RegisterExporter(pe) + pe, err := prometheus.NewExporter(prometheus.Options{ + Namespace: "revad", + }) + if err != nil { + return nil, errors.Wrap(err, "prometheus: error creating exporter") + } - // return &svc{prefix: conf.Prefix, h: pe}, nil + view.RegisterExporter(pe) - return &svc{prefix: conf.Prefix, h: promhttp.Handler()}, nil + return &svc{prefix: conf.Prefix, h: pe}, nil } type config struct { From 87aeaaae0c2b3a03abfdbeee7c5dcb5471520e5e Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Wed, 13 May 2020 13:32:58 +0200 Subject: [PATCH 16/25] Move metrics.toml to examples/metrics. --- examples/metrics/metrics.toml | 5 ++++- examples/oc-phoenix/metrics.toml | 7 ------- 2 files changed, 4 insertions(+), 8 deletions(-) delete mode 100644 examples/oc-phoenix/metrics.toml diff --git a/examples/metrics/metrics.toml b/examples/metrics/metrics.toml index f06751e8be..6caa3cc2ae 100644 --- a/examples/metrics/metrics.toml +++ b/examples/metrics/metrics.toml @@ -1,4 +1,7 @@ [shared] -jwt_secret = "mysecret" +jwt_secret = "Pive-Fumkiu4" [http.services.prometheus] + +[http] +address = "localhost:5550" \ No newline at end of file diff --git a/examples/oc-phoenix/metrics.toml b/examples/oc-phoenix/metrics.toml deleted file mode 100644 index 6caa3cc2ae..0000000000 --- a/examples/oc-phoenix/metrics.toml +++ /dev/null @@ -1,7 +0,0 @@ -[shared] -jwt_secret = "Pive-Fumkiu4" - -[http.services.prometheus] - -[http] -address = "localhost:5550" \ No newline at end of file From 95f66ca5bf00d14458ff2b270cd494db4e4328c6 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Fri, 22 May 2020 16:32:36 +0200 Subject: [PATCH 17/25] Make server listen on all interfaces. --- examples/metrics/metrics.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/metrics/metrics.toml b/examples/metrics/metrics.toml index 6caa3cc2ae..caa41975f7 100644 --- a/examples/metrics/metrics.toml +++ b/examples/metrics/metrics.toml @@ -4,4 +4,4 @@ jwt_secret = "Pive-Fumkiu4" [http.services.prometheus] [http] -address = "localhost:5550" \ No newline at end of file +address = "0.0.0.0:5550" \ No newline at end of file From 9aa3e8660c5fa44f8f3eea7fb6c54d339bddae0f Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Fri, 22 May 2020 16:33:47 +0200 Subject: [PATCH 18/25] Use opencensus.io for measures and view. --- pkg/metrics/metrics.go | 109 +++++++++++++++++++++++++++++------------ 1 file changed, 77 insertions(+), 32 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 4a4e7fa411..cf4788100b 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -18,55 +18,100 @@ package metrics +// This package defines site metrics measures and views based on opencensus.io + import ( + "context" "fmt" "math/rand" "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" ) func init() { - fmt.Printf("Init metrics\n") - getNumUsers() - getNumGroups() -} - -func getNumUsers() { - // here we must request the actual number of users from the site - // for now this sets a random dummy value - rand.Seed(time.Now().UnixNano()) + // trigger the actual metric provider functions go func() { + rand.Seed(time.Now().UnixNano()) for { - numUsersGauge.Set(float64(rand.Intn(10))) - time.Sleep(2 * time.Second) + getNumUsers() + getNumGroups() + getAmountStorage() + time.Sleep(4 * time.Second) } }() } +// Create the measures +var ( + NumUsersMeasure = stats.Int64("cs3_org_sciencemesh_site_total_num_users", "The total number of users within this site", stats.UnitDimensionless) + NumGroupsMeasure = stats.Int64("cs3_org_sciencemesh_site_total_num_groups", "The total number of groups within this site", stats.UnitDimensionless) + AmountStorageMeasure = stats.Int64("cs3_org_sciencemesh_site_total_amount_storage", "The total amount of storage used within this site", stats.UnitDimensionless) +) + +// initialize local dummy counters var ( - numUsersGauge = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "cs3_org_sciencemesh_site_total_num_users", - Help: "The total number of users within this site", - }) + numUsersCounter = int64(0) + amountStorageCounter = int64(0) ) +// getNumberUsers links to the underlying number of site users provider +func getNumUsers() { + ctx := context.Background() + // here we must request the actual number of site users + // for now this is a mockup: a number increasing over time + numUsersCounter += int64(rand.Intn(100)) + fmt.Printf("nrUsers = %v \n", numUsersCounter) + stats.Record(ctx, NumUsersMeasure.M(numUsersCounter)) +} + +// GetNumUsersView returns the number of site users measure view +func GetNumUsersView() *view.View { + return &view.View{ + Name: NumUsersMeasure.Name(), + Description: NumUsersMeasure.Description(), + Measure: NumUsersMeasure, + Aggregation: view.LastValue(), + } +} + +// getNumberGroups links to the underlying number of site groups provider func getNumGroups() { - // here we must request the actual number of groups from the site - // for now this sets a random dummy value - rand.Seed(time.Now().UnixNano()) - go func() { - for { - numGroupsGauge.Set(float64(rand.Intn(10))) - time.Sleep(2 * time.Second) - } - }() + ctx := context.Background() + // here we must request the actual number of site groups + // for now this is a mockup: a number changing over time + var numGroupsCounter = int64(rand.Intn(100)) + fmt.Printf("nrGroups = %v \n", numGroupsCounter) + stats.Record(ctx, NumGroupsMeasure.M(numGroupsCounter)) } -var ( - numGroupsGauge = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "cs3_org_sciencemesh_site_total_num_groups", - Help: "The total number of groups within this site", - }) -) +// GetNumGroupsView returns the number of site groups measure view +func GetNumGroupsView() *view.View { + return &view.View{ + Name: NumGroupsMeasure.Name(), + Description: NumGroupsMeasure.Description(), + Measure: NumGroupsMeasure, + Aggregation: view.LastValue(), + } +} + +// getAmountStorage links to the underlying amount of storage provider +func getAmountStorage() { + ctx := context.Background() + // here we must request the actual amount of storage used + // for now this is a mockup: a number increasing over time + amountStorageCounter += int64(rand.Intn(12865000)) + fmt.Printf("amountStorage = %v \n", amountStorageCounter) + stats.Record(ctx, AmountStorageMeasure.M(amountStorageCounter)) +} + +// GetAmountStorageView returns the amount of site storage measure view +func GetAmountStorageView() *view.View { + return &view.View{ + Name: AmountStorageMeasure.Name(), + Description: AmountStorageMeasure.Description(), + Measure: AmountStorageMeasure, + Aggregation: view.LastValue(), + } +} From 18fded32b04a5a49c10ce9518b53a1d673121318 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Fri, 22 May 2020 16:35:35 +0200 Subject: [PATCH 19/25] Register the desired measures' views. --- internal/http/services/prometheus/prometheus.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/internal/http/services/prometheus/prometheus.go b/internal/http/services/prometheus/prometheus.go index 4462e193c8..3c08da5c27 100644 --- a/internal/http/services/prometheus/prometheus.go +++ b/internal/http/services/prometheus/prometheus.go @@ -19,17 +19,21 @@ package prometheus import ( + "fmt" "net/http" + "reva/pkg/metrics" "contrib.go.opencensus.io/exporter/prometheus" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" + "github.com/rs/zerolog" "go.opencensus.io/stats/view" ) func init() { global.Register("prometheus", New) + fmt.Printf("metrics - %v \n", metrics.NumUsersMeasure.Description()) } // New returns a new prometheus service @@ -52,6 +56,13 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) view.RegisterExporter(pe) + // register the desired measures' views + view.Register( + metrics.GetNumUsersView(), + metrics.GetNumGroupsView(), + metrics.GetAmountStorageView(), + ) + return &svc{prefix: conf.Prefix, h: pe}, nil } From f4db96e1e9840fc74dfe1acb4dbb27d57c3c8f0b Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Mon, 25 May 2020 09:45:47 +0200 Subject: [PATCH 20/25] Comment edited --- pkg/metrics/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index cf4788100b..fa02785650 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -31,7 +31,7 @@ import ( ) func init() { - // trigger the actual metric provider functions + // call the actual metric provider functions for the latest metrics every 4th second go func() { rand.Seed(time.Now().UnixNano()) for { From 8097567eda52f796e09a966af36470cbc338a514 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Mon, 25 May 2020 13:50:19 +0200 Subject: [PATCH 21/25] Debug lines removed --- pkg/metrics/metrics.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index fa02785650..ac6d3f05c2 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -22,7 +22,6 @@ package metrics import ( "context" - "fmt" "math/rand" "time" @@ -62,7 +61,6 @@ func getNumUsers() { // here we must request the actual number of site users // for now this is a mockup: a number increasing over time numUsersCounter += int64(rand.Intn(100)) - fmt.Printf("nrUsers = %v \n", numUsersCounter) stats.Record(ctx, NumUsersMeasure.M(numUsersCounter)) } @@ -82,7 +80,6 @@ func getNumGroups() { // here we must request the actual number of site groups // for now this is a mockup: a number changing over time var numGroupsCounter = int64(rand.Intn(100)) - fmt.Printf("nrGroups = %v \n", numGroupsCounter) stats.Record(ctx, NumGroupsMeasure.M(numGroupsCounter)) } @@ -102,7 +99,6 @@ func getAmountStorage() { // here we must request the actual amount of storage used // for now this is a mockup: a number increasing over time amountStorageCounter += int64(rand.Intn(12865000)) - fmt.Printf("amountStorage = %v \n", amountStorageCounter) stats.Record(ctx, AmountStorageMeasure.M(amountStorageCounter)) } From 3b10f6625815dd26c842e8676fd5d999a49837b8 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Mon, 25 May 2020 13:52:17 +0200 Subject: [PATCH 22/25] Import fixed --- internal/http/services/prometheus/prometheus.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/http/services/prometheus/prometheus.go b/internal/http/services/prometheus/prometheus.go index 3c08da5c27..be02988862 100644 --- a/internal/http/services/prometheus/prometheus.go +++ b/internal/http/services/prometheus/prometheus.go @@ -21,7 +21,8 @@ package prometheus import ( "fmt" "net/http" - "reva/pkg/metrics" + + "github.com/cs3org/reva/pkg/metrics" "contrib.go.opencensus.io/exporter/prometheus" "github.com/cs3org/reva/pkg/rhttp/global" From 506132d157229571f98a45cee52d98cb9e55391d Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Mon, 25 May 2020 14:03:32 +0200 Subject: [PATCH 23/25] Use bytes as unit for storage amount. --- pkg/metrics/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index ac6d3f05c2..03e4690d0e 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -46,7 +46,7 @@ func init() { var ( NumUsersMeasure = stats.Int64("cs3_org_sciencemesh_site_total_num_users", "The total number of users within this site", stats.UnitDimensionless) NumGroupsMeasure = stats.Int64("cs3_org_sciencemesh_site_total_num_groups", "The total number of groups within this site", stats.UnitDimensionless) - AmountStorageMeasure = stats.Int64("cs3_org_sciencemesh_site_total_amount_storage", "The total amount of storage used within this site", stats.UnitDimensionless) + AmountStorageMeasure = stats.Int64("cs3_org_sciencemesh_site_total_amount_storage", "The total amount of storage used within this site", stats.UnitBytes) ) // initialize local dummy counters From fdbf46e455cdf625e32152b6e0d8386fc2100761 Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Mon, 25 May 2020 14:08:51 +0200 Subject: [PATCH 24/25] Catch error and return. --- internal/http/services/prometheus/prometheus.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/http/services/prometheus/prometheus.go b/internal/http/services/prometheus/prometheus.go index be02988862..5b39d9eff3 100644 --- a/internal/http/services/prometheus/prometheus.go +++ b/internal/http/services/prometheus/prometheus.go @@ -58,11 +58,13 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) view.RegisterExporter(pe) // register the desired measures' views - view.Register( + if err = view.Register( metrics.GetNumUsersView(), metrics.GetNumGroupsView(), metrics.GetAmountStorageView(), - ) + ); err != nil { + return nil, errors.Wrap(err, "prometheus: error registering exporter") + } return &svc{prefix: conf.Prefix, h: pe}, nil } From 571e1fb94d9e4845fed3569dacc264831157a71e Mon Sep 17 00:00:00 2001 From: Antoon Prins Date: Mon, 25 May 2020 15:18:20 +0200 Subject: [PATCH 25/25] Debug line removed. --- internal/http/services/prometheus/prometheus.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/http/services/prometheus/prometheus.go b/internal/http/services/prometheus/prometheus.go index 5b39d9eff3..eb669408bd 100644 --- a/internal/http/services/prometheus/prometheus.go +++ b/internal/http/services/prometheus/prometheus.go @@ -19,7 +19,6 @@ package prometheus import ( - "fmt" "net/http" "github.com/cs3org/reva/pkg/metrics" @@ -34,7 +33,6 @@ import ( func init() { global.Register("prometheus", New) - fmt.Printf("metrics - %v \n", metrics.NumUsersMeasure.Description()) } // New returns a new prometheus service