From 083bda9866f675442f4606cbd96e765197f92ac4 Mon Sep 17 00:00:00 2001 From: "lukasz.widera@vshn.ch" Date: Mon, 24 Jun 2024 10:13:10 +0200 Subject: [PATCH 1/3] refactoring SPKS and adding base metrics --- go.mod | 6 +- go.sum | 6 + main.go | 59 ++++++- pkg/cloudscale/objectstorage.go | 8 +- pkg/cmd/cloudscale.go | 7 +- pkg/cmd/exoscale.go | 9 +- pkg/cmd/spks.go | 291 +++++++++++++++++--------------- pkg/exoscale/objectstorage.go | 11 +- pkg/odoo/odoo.go | 11 +- 9 files changed, 251 insertions(+), 157 deletions(-) diff --git a/go.mod b/go.mod index 725d839..7d35718 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/exoscale/egoscale v0.90.1 github.com/go-logr/logr v1.3.0 github.com/go-logr/zapr v1.3.0 - github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/client_golang v1.19.1 github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.24.4 github.com/vshn/provider-cloudscale v0.5.0 @@ -79,13 +79,13 @@ require ( golang.org/x/crypto v0.18.0 // indirect golang.org/x/exp v0.0.0-20230307190834-24139beb5833 // indirect golang.org/x/net v0.20.0 // indirect - golang.org/x/sys v0.16.0 // indirect + golang.org/x/sys v0.17.0 // indirect golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.32.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index f6a4d69..cd19c1d 100644 --- a/go.sum +++ b/go.sum @@ -358,6 +358,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= @@ -626,6 +628,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -814,6 +818,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/main.go b/main.go index dc83c9e..e6cdfa7 100644 --- a/main.go +++ b/main.go @@ -3,12 +3,16 @@ package main import ( "context" "fmt" + "net/http" "os" "os/signal" "runtime" "syscall" "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/urfave/cli/v2" "github.com/vshn/billing-collector-cloudservices/pkg/cmd" "github.com/vshn/billing-collector-cloudservices/pkg/log" @@ -22,6 +26,40 @@ var ( appName = "billing-collector-cloudservices" appLongName = "Metrics collector which gathers metrics information for cloud services" + + odooFailed = promauto.NewCounter(prometheus.CounterOpts{ + Name: "billing_cloud_collector_http_requests_odoo_failed_total", + Help: "Total number of failed HTTP requests to Odoo", + }) + odooSucceeded = promauto.NewCounter(prometheus.CounterOpts{ + Name: "billing_cloud_collector_http_requests_odoo_succeeded_total", + Help: "Total number of successful HTTP requests to Odoo", + }) + + providerFailed = promauto.NewCounter(prometheus.CounterOpts{ + Name: "billing_cloud_collector_http_requests_provider_failed_total", + Help: "Total number of failed HTTP requests to the cloud provider", + }) + + providerSucceeded = promauto.NewCounter(prometheus.CounterOpts{ + Name: "billing_cloud_collector_http_requests_provider_succeeded_total", + Help: "Total number of successful HTTP requests to the cloud provider", + }) + + providerMetrics = map[string]prometheus.Counter{ + "providerFailed": providerFailed, + "providerSucceeded": providerSucceeded, + } + + odooMetrics = map[string]prometheus.Counter{ + "odooFailed": odooFailed, + "odooSucceeded": odooSucceeded, + } + + allMetrics = map[string]map[string]prometheus.Counter{ + "odooMetrics": odooMetrics, + "providerMetrics": providerMetrics, + } ) func init() { @@ -32,12 +70,23 @@ func init() { func main() { ctx, stop, app := newApp() defer stop() + + go func() { + http.Handle("/metrics", promhttp.Handler()) + err := http.ListenAndServe(":2112", nil) + if err != nil { + fmt.Println("Error starting prometheus server: ", err.Error()) + } + os.Exit(1) + }() + err := app.RunContext(ctx, os.Args) // If required flags aren't set, it will return with error before we could set up logging if err != nil { _, _ = fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } + } func newApp() (context.Context, context.CancelFunc, *cli.App) { @@ -45,6 +94,9 @@ func newApp() (context.Context, context.CancelFunc, *cli.App) { logLevel int logFormat string ) + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + app := &cli.App{ Name: appName, Usage: appLongName, @@ -120,9 +172,9 @@ func newApp() (context.Context, context.CancelFunc, *cli.App) { return nil }, Commands: []*cli.Command{ - cmd.ExoscaleCmds(), - cmd.CloudscaleCmds(), - cmd.SpksCMD(), + cmd.ExoscaleCmds(allMetrics), + cmd.CloudscaleCmds(allMetrics), + cmd.SpksCMD(allMetrics, ctx), }, ExitErrHandler: func(c *cli.Context, err error) { if err != nil { @@ -132,6 +184,5 @@ func newApp() (context.Context, context.CancelFunc, *cli.App) { }, } - ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) return ctx, stop, app } diff --git a/pkg/cloudscale/objectstorage.go b/pkg/cloudscale/objectstorage.go index b882c4f..2b2f12d 100644 --- a/pkg/cloudscale/objectstorage.go +++ b/pkg/cloudscale/objectstorage.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/vshn/billing-collector-cloudservices/pkg/controlAPI" "github.com/vshn/billing-collector-cloudservices/pkg/kubernetes" "github.com/vshn/billing-collector-cloudservices/pkg/log" @@ -30,13 +31,14 @@ type ObjectStorage struct { clusterId string cloudZone string uomMapping map[string]string + providerMetrics map[string]prometheus.Counter } const ( namespaceLabel = "crossplane.io/claim-namespace" ) -func NewObjectStorage(client *cloudscale.Client, k8sClient k8s.Client, controlApiClient k8s.Client, salesOrder, clusterId string, cloudZone string, uomMapping map[string]string) (*ObjectStorage, error) { +func NewObjectStorage(client *cloudscale.Client, k8sClient k8s.Client, controlApiClient k8s.Client, salesOrder, clusterId string, cloudZone string, uomMapping map[string]string, providerMetrics map[string]prometheus.Counter) (*ObjectStorage, error) { return &ObjectStorage{ client: client, k8sClient: k8sClient, @@ -45,6 +47,7 @@ func NewObjectStorage(client *cloudscale.Client, k8sClient k8s.Client, controlAp clusterId: clusterId, cloudZone: cloudZone, uomMapping: uomMapping, + providerMetrics: providerMetrics, }, nil } @@ -56,7 +59,10 @@ func (o *ObjectStorage) GetMetrics(ctx context.Context, billingDate time.Time) ( bucketMetricsRequest := cloudscale.BucketMetricsRequest{Start: billingDate, End: billingDate} bucketMetrics, err := o.client.Metrics.GetBucketMetrics(ctx, &bucketMetricsRequest) if err != nil { + o.providerMetrics["providerFailed"].Inc() return nil, err + } else { + o.providerMetrics["providerSucceeded"].Inc() } // Fetch organisations in case salesOrder is missing diff --git a/pkg/cmd/cloudscale.go b/pkg/cmd/cloudscale.go index 3470e0d..38d4a16 100644 --- a/pkg/cmd/cloudscale.go +++ b/pkg/cmd/cloudscale.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/vshn/billing-collector-cloudservices/pkg/odoo" "github.com/cloudscale-ch/cloudscale-go-sdk/v2" @@ -19,7 +20,7 @@ import ( const defaultTextForRequiredFlags = "" const defaultTextForOptionalFlags = "" -func CloudscaleCmds() *cli.Command { +func CloudscaleCmds(allMetrics map[string]map[string]prometheus.Counter) *cli.Command { var ( apiToken string kubeconfig string @@ -102,14 +103,14 @@ func CloudscaleCmds() *cli.Command { return fmt.Errorf("k8s control client: %w", err) } - odooClient := odoo.NewOdooAPIClient(c.Context, odooURL, odooOauthTokenURL, odooClientId, odooClientSecret, logger) + odooClient := odoo.NewOdooAPIClient(c.Context, odooURL, odooOauthTokenURL, odooClientId, odooClientSecret, logger, allMetrics["odooMetrics"]) location, err := time.LoadLocation("Europe/Zurich") if err != nil { return fmt.Errorf("load loaction: %w", err) } - o, err := cs.NewObjectStorage(cloudscaleClient, k8sClient, k8sControlClient, salesOrder, clusterId, cloudZone, mapping) + o, err := cs.NewObjectStorage(cloudscaleClient, k8sClient, k8sControlClient, salesOrder, clusterId, cloudZone, mapping, allMetrics["providerMetrics"]) if err != nil { return fmt.Errorf("object storage: %w", err) } diff --git a/pkg/cmd/exoscale.go b/pkg/cmd/exoscale.go index bb8c6b4..0dba485 100644 --- a/pkg/cmd/exoscale.go +++ b/pkg/cmd/exoscale.go @@ -6,6 +6,7 @@ import ( "sync" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/vshn/billing-collector-cloudservices/pkg/odoo" "github.com/urfave/cli/v2" @@ -19,7 +20,7 @@ func addCommandName(c *cli.Context) error { return nil } -func ExoscaleCmds() *cli.Command { +func ExoscaleCmds(allMetrics map[string]map[string]prometheus.Counter) *cli.Command { var ( secret string accessKey string @@ -112,14 +113,14 @@ func ExoscaleCmds() *cli.Command { return fmt.Errorf("k8s control client: %w", err) } - odooClient := odoo.NewOdooAPIClient(c.Context, odooURL, odooOauthTokenURL, odooClientId, odooClientSecret, logger) + odooClient := odoo.NewOdooAPIClient(c.Context, odooURL, odooOauthTokenURL, odooClientId, odooClientSecret, logger, allMetrics["odooMetrics"]) if collectInterval < 1 || collectInterval > 23 { // Set to run once a day after billingHour in case the collectInterval is out of boundaries collectInterval = 23 } - o, err := exoscale.NewObjectStorage(exoscaleClient, k8sClient, k8sControlClient, salesOrder, clusterId, cloudZone, mapping) + o, err := exoscale.NewObjectStorage(exoscaleClient, k8sClient, k8sControlClient, salesOrder, clusterId, cloudZone, mapping, allMetrics["providerMetrics"]) if err != nil { return fmt.Errorf("objectbucket service: %w", err) } @@ -192,7 +193,7 @@ func ExoscaleCmds() *cli.Command { return fmt.Errorf("k8s control client: %w", err) } - odooClient := odoo.NewOdooAPIClient(c.Context, odooURL, odooOauthTokenURL, odooClientId, odooClientSecret, logger) + odooClient := odoo.NewOdooAPIClient(c.Context, odooURL, odooOauthTokenURL, odooClientId, odooClientSecret, logger, allMetrics["odooMetrics"]) if collectInterval < 1 || collectInterval > 24 { // Set to run once a day after billingHour in case the collectInterval is out of boundaries diff --git a/pkg/cmd/spks.go b/pkg/cmd/spks.go index e10079e..e7ccd1d 100644 --- a/pkg/cmd/spks.go +++ b/pkg/cmd/spks.go @@ -2,41 +2,41 @@ package cmd import ( "context" - "fmt" - "os" - "os/signal" "time" "github.com/go-logr/logr" "github.com/prometheus/client_golang/api" v1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/urfave/cli/v2" "github.com/vshn/billing-collector-cloudservices/pkg/log" "github.com/vshn/billing-collector-cloudservices/pkg/odoo" ) -func SpksCMD() *cli.Command { - var ( - prometheusQueryArr = [4]string{ - "count(max_over_time(crossplane_resource_info{kind=\"compositemariadbinstances\", service_level=\"standard\"}[1d:1d]))", - "count(max_over_time(crossplane_resource_info{kind=\"compositemariadbinstances\", service_level=\"premium\"}[1d:1d]))", - "count(max_over_time(crossplane_resource_info{kind=\"compositeredisinstances\", service_level=\"standard\"}[1d:1d]))", - "count(max_over_time(crossplane_resource_info{kind=\"compositeredisinstances\", service_level=\"premium\"}[1d:1d]))", - } - odooURL string - odooOauthTokenURL string - odooClientId string - odooClientSecret string - salesOrder string - prometheusURL string - UnitID string - ) +var ( + prometheusQueryArr = [4]string{ + "count(max_over_time(crossplane_resource_info{kind=\"compositemariadbinstances\", service_level=\"standard\"}[1d:1d]))", + "count(max_over_time(crossplane_resource_info{kind=\"compositemariadbinstances\", service_level=\"premium\"}[1d:1d]))", + "count(max_over_time(crossplane_resource_info{kind=\"compositeredisinstances\", service_level=\"standard\"}[1d:1d]))", + "count(max_over_time(crossplane_resource_info{kind=\"compositeredisinstances\", service_level=\"premium\"}[1d:1d]))", + } + odooURL string + odooOauthTokenURL string + odooClientId string + odooClientSecret string + salesOrder string + prometheusURL string + UnitID string + days int +) + +func SpksCMD(allMetrics map[string]map[string]prometheus.Counter, ctx context.Context) *cli.Command { return &cli.Command{ Name: "spks", - Usage: "Collect metrics from spks", + Usage: "Collect metrics from spks.", Before: addCommandName, Flags: []cli.Flag{ &cli.StringFlag{Name: "odoo-url", Usage: "URL of the Odoo Metered Billing API", @@ -53,144 +53,151 @@ func SpksCMD() *cli.Command { EnvVars: []string{"PROMETHEUS_URL"}, Destination: &prometheusURL, Required: false, DefaultText: defaultTextForRequiredFlags, Value: "http://prometheus-monitoring-application.monitoring-application.svc.cluster.local:9090"}, &cli.StringFlag{Name: "unit-id", Usage: "Metered Billing UoM ID for the consumed units", EnvVars: []string{"UNIT_ID"}, Destination: &UnitID, Required: false, DefaultText: defaultTextForRequiredFlags, Value: "uom_uom_68_b1811ca1"}, + &cli.IntFlag{Name: "days", Usage: "Days of metrics to fetch since today, set to 0 to get current metrics", + EnvVars: []string{"DAYS"}, Destination: &days, Value: 0, Required: false, DefaultText: defaultTextForOptionalFlags}, }, Action: func(c *cli.Context) error { + ctxx, cancel := context.WithCancel(ctx) + defer cancel() logger := log.Logger(c.Context) logger.Info("starting spks data collector") - location, err := time.LoadLocation("Europe/Zurich") - if err != nil { - return fmt.Errorf("load location: %w", err) - } - // this variable is necessary to query Prometheus, with timerange [1d:1d] it returns data from 1 day up to midnight - startOfToday := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, location) - startYesterdayAbsolute := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day()-1, 0, 0, 0, 0, location).In(time.UTC) - endYesterdayAbsolute := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day()-1, 23, 59, 59, 0, location).In(time.UTC) - - logger.Info("Getting specific metric from thanos") - - client, err := api.NewClient(api.Config{ - Address: prometheusURL, - }) - if err != nil { - logger.Error(err, "Error creating Prometheus client") - return err - } - - v1api := v1.NewAPI(client) - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - // send this value to odoo - - mariadbStandard, err := QueryPrometheus(ctx, v1api, prometheusQueryArr[0], logger, startOfToday) - if err != nil { - logger.Error(err, "Error querying Prometheus") - return err - } - mariadbPremium, err := QueryPrometheus(ctx, v1api, prometheusQueryArr[1], logger, startOfToday) - if err != nil { - logger.Error(err, "Error querying Prometheus") - return err - } - redisStandard, err := QueryPrometheus(ctx, v1api, prometheusQueryArr[2], logger, startOfToday) - if err != nil { - logger.Error(err, "Error querying Prometheus") - return err - } - redisPremium, err := QueryPrometheus(ctx, v1api, prometheusQueryArr[3], logger, startOfToday) - if err != nil { - logger.Error(err, "Error querying Prometheus") - return err - } - - odooClient := odoo.NewOdooAPIClient(c.Context, odooURL, odooOauthTokenURL, odooClientId, odooClientSecret, logger) - - billingRecords := []odoo.OdooMeteredBillingRecord{ - { - ProductID: "appcat-spks-mariadb-standard", - InstanceID: "mariadb-standard", - SalesOrder: salesOrder, - UnitID: UnitID, - ConsumedUnits: float64(mariadbStandard), - TimeRange: odoo.TimeRange{ - From: startYesterdayAbsolute, - To: endYesterdayAbsolute, - }, - }, - { - ProductID: "appcat-spks-mariadb-premium", - InstanceID: "mariadb-premium", - SalesOrder: salesOrder, - UnitID: UnitID, - ConsumedUnits: float64(mariadbPremium), - TimeRange: odoo.TimeRange{ - From: startYesterdayAbsolute, - To: endYesterdayAbsolute, - }, - }, - { - ProductID: "appcat-spks-redis-standard", - InstanceID: "redis-standard", - SalesOrder: salesOrder, - UnitID: UnitID, - ConsumedUnits: float64(redisStandard), - TimeRange: odoo.TimeRange{ - From: startYesterdayAbsolute, - To: endYesterdayAbsolute, - }, - }, - { - ProductID: "appcat-spks-redis-premium", - InstanceID: "redis-premium", - SalesOrder: salesOrder, - UnitID: UnitID, - ConsumedUnits: float64(redisPremium), - TimeRange: odoo.TimeRange{ - From: startYesterdayAbsolute, - To: endYesterdayAbsolute, - }, - }, - } - ticker := time.NewTicker(24 * time.Hour) - sigint := make(chan os.Signal, 1) - signal.Notify(sigint, os.Interrupt) - - // ensure first data export after restart - // I want to avoid situation when data is not exported because of constant restarts - err = odooClient.SendData(billingRecords) - if err != nil { - logger.Error(err, "can't export data to Odoo") - return err - } - for loop := true; loop; { + runSPKSBilling(prometheusURL, prometheusQueryArr, logger, allMetrics, salesOrder, UnitID, c.Context) + + for { select { + case <-ctxx.Done(): + logger.Info("Received Context cancellation, exiting...") + return nil case <-ticker.C: // this runs every 24 hours after program start - err = odooClient.SendData(billingRecords) - if err != nil { - logger.Error(err, "can't export data to Odoo") - return err - } - case <-sigint: - loop = false - // this one breaks select{} statement - // immediately and finish function - break + runSPKSBilling(prometheusURL, prometheusQueryArr, logger, allMetrics, salesOrder, UnitID, c.Context) } } - return nil }, } } -func QueryPrometheus(ctx context.Context, v1api v1.API, query string, logger logr.Logger, absoluteBeginningTime time.Time) (int, error) { +func runSPKSBilling(prometheusURL string, prometheusQueryArr [4]string, logger logr.Logger, allMetrics map[string]map[string]prometheus.Counter, salesOrder string, UnitID string, c context.Context) { + location, err := time.LoadLocation("Europe/Zurich") + if err != nil { + allMetrics["odooMetrics"]["odooFailed"].Inc() + } + now := time.Now().In(location) + // this variable is necessary to query Prometheus, with timerange [1d:1d] it returns data from 1 day up to midnight + startOfToday := time.Date(now.Year(), now.Month(), now.Day()-days, 0, 0, 0, 0, location) + startYesterdayAbsolute := time.Date(now.Year(), now.Month(), now.Day()-days-1, 0, 0, 0, 0, location).In(time.UTC) + endYesterdayAbsolute := startYesterdayAbsolute.Add(24 * time.Hour) + + logger.Info("Running SPKS billing with such timeranges: ", "startOfToday", startOfToday, "startYesterdayAbsolute", startYesterdayAbsolute, "endYesterdayAbsolute", endYesterdayAbsolute) + + odooClient := odoo.NewOdooAPIClient(c, odooURL, odooOauthTokenURL, odooClientId, odooClientSecret, logger, allMetrics["odooMetrics"]) + + mariadbStandard, mariadbPremium, redisStandard, redisPremium, err := getDatabasesCounts(prometheusURL, prometheusQueryArr, logger, startOfToday, allMetrics) + if err != nil { + logger.Error(err, "Error getting database counts") + } + + billingRecords := generateBillingRecords(salesOrder, UnitID, startYesterdayAbsolute, endYesterdayAbsolute, mariadbStandard, mariadbPremium, redisStandard, redisPremium) + + err = odooClient.SendData(billingRecords) + if err != nil { + logger.Error(err, "Error sending data to Odoo API") + } +} + +func generateBillingRecords(salesOrder string, UnitID string, startYesterdayAbsolute time.Time, endYesterdayAbsolute time.Time, mariadbStandard int, mariadbPremium int, redisStandard int, redisPremium int) []odoo.OdooMeteredBillingRecord { + timerange := odoo.TimeRange{ + From: startYesterdayAbsolute, + To: endYesterdayAbsolute, + } + + billingRecords := []odoo.OdooMeteredBillingRecord{ + { + ProductID: "appcat-spks-mariadb-standard", + InstanceID: "mariadb-standard", + SalesOrder: salesOrder, + UnitID: UnitID, + ConsumedUnits: float64(mariadbStandard), + TimeRange: timerange, + }, + { + ProductID: "appcat-spks-mariadb-premium", + InstanceID: "mariadb-premium", + SalesOrder: salesOrder, + UnitID: UnitID, + ConsumedUnits: float64(mariadbPremium), + TimeRange: timerange, + }, + { + ProductID: "appcat-spks-redis-standard", + InstanceID: "redis-standard", + SalesOrder: salesOrder, + UnitID: UnitID, + ConsumedUnits: float64(redisStandard), + TimeRange: timerange, + }, + { + ProductID: "appcat-spks-redis-premium", + InstanceID: "redis-premium", + SalesOrder: salesOrder, + UnitID: UnitID, + ConsumedUnits: float64(redisPremium), + TimeRange: timerange, + }, + } + + return billingRecords +} + +func getDatabasesCounts(prometheusURL string, prometheusQueryArr [4]string, logger logr.Logger, startOfToday time.Time, allMetrics map[string]map[string]prometheus.Counter) (int, int, int, int, error) { + + client, err := api.NewClient(api.Config{ + Address: prometheusURL, + }) + if err != nil { + logger.Error(err, "Error creating Prometheus client") + } + + v1api := v1.NewAPI(client) + ctxx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + mariadbStandard, err := QueryPrometheus(ctxx, v1api, prometheusQueryArr[0], logger, startOfToday, allMetrics["providerMetrics"]) + if err != nil { + return -1, -1, -1, -1, err + } + + mariadbPremium, err := QueryPrometheus(ctxx, v1api, prometheusQueryArr[1], logger, startOfToday, allMetrics["providerMetrics"]) + if err != nil { + return -1, -1, -1, -1, err + } + + redisStandard, err := QueryPrometheus(ctxx, v1api, prometheusQueryArr[2], logger, startOfToday, allMetrics["providerMetrics"]) + if err != nil { + return -1, -1, -1, -1, err + } + + redisPremium, err := QueryPrometheus(ctxx, v1api, prometheusQueryArr[3], logger, startOfToday, allMetrics["providerMetrics"]) + if err != nil { + return -1, -1, -1, -1, err + } + + return mariadbStandard, mariadbPremium, redisStandard, redisPremium, nil +} + +func QueryPrometheus(ctx context.Context, v1api v1.API, query string, logger logr.Logger, absoluteBeginningTime time.Time, providerMetrics map[string]prometheus.Counter) (int, error) { result, warnings, err := v1api.Query(ctx, query, absoluteBeginningTime, v1.WithTimeout(5*time.Second)) if err != nil { + providerMetrics["providerFailed"].Inc() logger.Error(err, "Error querying Prometheus") - return 0, err + return -1, err } + + providerMetrics["providerSucceeded"].Inc() + if len(warnings) > 0 { logger.Info("Warnings", "warnings from Prometheus query", warnings) } @@ -199,10 +206,14 @@ func QueryPrometheus(ctx context.Context, v1api v1.API, query string, logger log case model.ValVector: vectorVal := result.(model.Vector) if len(vectorVal) != 1 { - return 0, fmt.Errorf("expected 1 result, got %d", len(vectorVal)) + logger.Error(err, "Result vector length is not 1, prometheus query failed and returned: ", "result", vectorVal) + providerMetrics["providerFailed"].Inc() + return -1, err } default: - return 0, fmt.Errorf("result type is not Vector: %s", result.Type()) + logger.Error(err, "result type is not Vector: ", "result", result) + providerMetrics["providerFailed"].Inc() + return -1, err } return int(result.(model.Vector)[0].Value), nil diff --git a/pkg/exoscale/objectstorage.go b/pkg/exoscale/objectstorage.go index 8f9c819..3733258 100644 --- a/pkg/exoscale/objectstorage.go +++ b/pkg/exoscale/objectstorage.go @@ -7,6 +7,7 @@ import ( egoscale "github.com/exoscale/egoscale/v2" "github.com/exoscale/egoscale/v2/oapi" + "github.com/prometheus/client_golang/prometheus" "github.com/vshn/billing-collector-cloudservices/pkg/controlAPI" "github.com/vshn/billing-collector-cloudservices/pkg/exofixtures" "github.com/vshn/billing-collector-cloudservices/pkg/kubernetes" @@ -30,6 +31,7 @@ type ObjectStorage struct { clusterId string cloudZone string uomMapping map[string]string + providerMetrics map[string]prometheus.Counter } // BucketDetail a k8s bucket object with relevant data @@ -38,7 +40,7 @@ type BucketDetail struct { } // NewObjectStorage creates an ObjectStorage with the initial setup -func NewObjectStorage(exoscaleClient *egoscale.Client, k8sClient k8s.Client, controlApiClient k8s.Client, salesOrder, clusterId string, cloudZone string, uomMapping map[string]string) (*ObjectStorage, error) { +func NewObjectStorage(exoscaleClient *egoscale.Client, k8sClient k8s.Client, controlApiClient k8s.Client, salesOrder, clusterId string, cloudZone string, uomMapping map[string]string, providerMetrics map[string]prometheus.Counter) (*ObjectStorage, error) { return &ObjectStorage{ k8sClient: k8sClient, exoscaleClient: exoscaleClient, @@ -47,13 +49,17 @@ func NewObjectStorage(exoscaleClient *egoscale.Client, k8sClient k8s.Client, con clusterId: clusterId, cloudZone: cloudZone, uomMapping: uomMapping, + providerMetrics: providerMetrics, }, nil } func (o *ObjectStorage) GetMetrics(ctx context.Context) ([]odoo.OdooMeteredBillingRecord, error) { detail, err := o.fetchManagedBucketsAndNamespaces(ctx) if err != nil { + o.providerMetrics["providerFailed"].Inc() return nil, fmt.Errorf("fetchManagedBucketsAndNamespaces: %w", err) + } else { + o.providerMetrics["providerSucceeded"].Inc() } metrics, err := o.getBucketUsage(ctx, detail) @@ -71,7 +77,10 @@ func (o *ObjectStorage) getBucketUsage(ctx context.Context, bucketDetails []Buck resp, err := o.exoscaleClient.ListSosBucketsUsageWithResponse(ctx) if err != nil { + o.providerMetrics["providerFailed"].Inc() return nil, err + } else { + o.providerMetrics["providerSucceeded"].Inc() } odooMetrics, err := o.getOdooMeteredBillingRecords(ctx, *resp.JSON200.SosBucketsUsage, bucketDetails) diff --git a/pkg/odoo/odoo.go b/pkg/odoo/odoo.go index a8015b0..8348c29 100644 --- a/pkg/odoo/odoo.go +++ b/pkg/odoo/odoo.go @@ -11,6 +11,7 @@ import ( "time" "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" "golang.org/x/oauth2/clientcredentials" ) @@ -25,6 +26,7 @@ type OdooAPIClient struct { odooURL string logger logr.Logger oauthClient *http.Client + odooMetrics map[string]prometheus.Counter } type apiObject struct { @@ -55,7 +57,7 @@ func (t *TimeRange) UnmarshalJSON([]byte) error { return errors.New("Not implemented") } -func NewOdooAPIClient(ctx context.Context, odooURL string, oauthTokenURL string, oauthClientId string, oauthClientSecret string, logger logr.Logger) *OdooAPIClient { +func NewOdooAPIClient(ctx context.Context, odooURL string, oauthTokenURL string, oauthClientId string, oauthClientSecret string, logger logr.Logger, odooMetrics map[string]prometheus.Counter) *OdooAPIClient { oauthConfig := clientcredentials.Config{ ClientID: oauthClientId, ClientSecret: oauthClientSecret, @@ -66,6 +68,7 @@ func NewOdooAPIClient(ctx context.Context, odooURL string, oauthTokenURL string, odooURL: odooURL, logger: logger, oauthClient: oauthClient, + odooMetrics: odooMetrics, } } @@ -79,14 +82,20 @@ func (c OdooAPIClient) SendData(data []OdooMeteredBillingRecord) error { } resp, err := c.oauthClient.Post(c.odooURL, "application/json", bytes.NewBuffer(str)) if err != nil { + c.odooMetrics["odooFailed"].Inc() return err } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) c.logger.Info("Records sent to Odoo API", "status", resp.Status, "body", string(body), "numberOfRecords", len(data)) if resp.StatusCode != 200 { + c.odooMetrics["odooFailed"].Inc() return errors.New(fmt.Sprintf("API error when sending records to Odoo:\n%s", body)) + } else { + c.odooMetrics["odooSucceeded"].Inc() } return nil From 08e72e882d7cf06b561fb6bf06ac7dcd6bb0bf6e Mon Sep 17 00:00:00 2001 From: "lukasz.widera@vshn.ch" Date: Fri, 5 Jul 2024 10:47:26 +0200 Subject: [PATCH 2/3] adjusting timezone shift --- pkg/cmd/spks.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/cmd/spks.go b/pkg/cmd/spks.go index e7ccd1d..8a6d5c8 100644 --- a/pkg/cmd/spks.go +++ b/pkg/cmd/spks.go @@ -81,6 +81,7 @@ func SpksCMD(allMetrics map[string]map[string]prometheus.Counter, ctx context.Co } func runSPKSBilling(prometheusURL string, prometheusQueryArr [4]string, logger logr.Logger, allMetrics map[string]map[string]prometheus.Counter, salesOrder string, UnitID string, c context.Context) { + var startYesterdayAbsolute time.Time location, err := time.LoadLocation("Europe/Zurich") if err != nil { allMetrics["odooMetrics"]["odooFailed"].Inc() @@ -88,11 +89,17 @@ func runSPKSBilling(prometheusURL string, prometheusQueryArr [4]string, logger l now := time.Now().In(location) // this variable is necessary to query Prometheus, with timerange [1d:1d] it returns data from 1 day up to midnight startOfToday := time.Date(now.Year(), now.Month(), now.Day()-days, 0, 0, 0, 0, location) - startYesterdayAbsolute := time.Date(now.Year(), now.Month(), now.Day()-days-1, 0, 0, 0, 0, location).In(time.UTC) - endYesterdayAbsolute := startYesterdayAbsolute.Add(24 * time.Hour) - logger.Info("Running SPKS billing with such timeranges: ", "startOfToday", startOfToday, "startYesterdayAbsolute", startYesterdayAbsolute, "endYesterdayAbsolute", endYesterdayAbsolute) + if days != 0 { + startYesterdayAbsolute = time.Date(now.Year(), now.Month(), now.Day()-days, 0, 0, 0, 0, location).In(time.UTC) + } else { + startYesterdayAbsolute = time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, location).In(time.UTC) + } + + endYesterdayAbsolute := startYesterdayAbsolute.Add(24 * time.Hour) + logger.Info("Running SPKS billing with such timeranges: ", "startOfToday", startOfToday, "startYesterdayAbsolute", startYesterdayAbsolute.Local(), "endYesterdayAbsolute", endYesterdayAbsolute.Local()) + return odooClient := odoo.NewOdooAPIClient(c, odooURL, odooOauthTokenURL, odooClientId, odooClientSecret, logger, allMetrics["odooMetrics"]) mariadbStandard, mariadbPremium, redisStandard, redisPremium, err := getDatabasesCounts(prometheusURL, prometheusQueryArr, logger, startOfToday, allMetrics) From b86128595d3900f2ae322e38e77ebb8564f1e6c7 Mon Sep 17 00:00:00 2001 From: "lukasz.widera@vshn.ch" Date: Fri, 5 Jul 2024 11:17:06 +0200 Subject: [PATCH 3/3] fixing timestamp offsets --- pkg/cmd/spks.go | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/pkg/cmd/spks.go b/pkg/cmd/spks.go index 8a6d5c8..31e6f50 100644 --- a/pkg/cmd/spks.go +++ b/pkg/cmd/spks.go @@ -64,7 +64,12 @@ func SpksCMD(allMetrics map[string]map[string]prometheus.Counter, ctx context.Co ticker := time.NewTicker(24 * time.Hour) - runSPKSBilling(prometheusURL, prometheusQueryArr, logger, allMetrics, salesOrder, UnitID, c.Context) + daysChannel := make(chan int, 1) + if days != 0 { + daysChannel <- days + } else { + runSPKSBilling(prometheusURL, prometheusQueryArr, logger, allMetrics, salesOrder, UnitID, c.Context) + } for { select { @@ -74,6 +79,12 @@ func SpksCMD(allMetrics map[string]map[string]prometheus.Counter, ctx context.Co case <-ticker.C: // this runs every 24 hours after program start runSPKSBilling(prometheusURL, prometheusQueryArr, logger, allMetrics, salesOrder, UnitID, c.Context) + case <-daysChannel: + runSPKSBilling(prometheusURL, prometheusQueryArr, logger, allMetrics, salesOrder, UnitID, c.Context) + if days > 0 { + days-- + daysChannel <- days + } } } }, @@ -81,7 +92,7 @@ func SpksCMD(allMetrics map[string]map[string]prometheus.Counter, ctx context.Co } func runSPKSBilling(prometheusURL string, prometheusQueryArr [4]string, logger logr.Logger, allMetrics map[string]map[string]prometheus.Counter, salesOrder string, UnitID string, c context.Context) { - var startYesterdayAbsolute time.Time + // var startYesterdayAbsolute time.Time location, err := time.LoadLocation("Europe/Zurich") if err != nil { allMetrics["odooMetrics"]["odooFailed"].Inc() @@ -89,17 +100,12 @@ func runSPKSBilling(prometheusURL string, prometheusQueryArr [4]string, logger l now := time.Now().In(location) // this variable is necessary to query Prometheus, with timerange [1d:1d] it returns data from 1 day up to midnight startOfToday := time.Date(now.Year(), now.Month(), now.Day()-days, 0, 0, 0, 0, location) - - if days != 0 { - startYesterdayAbsolute = time.Date(now.Year(), now.Month(), now.Day()-days, 0, 0, 0, 0, location).In(time.UTC) - } else { - startYesterdayAbsolute = time.Date(now.Year(), now.Month(), now.Day()-1, 0, 0, 0, 0, location).In(time.UTC) - } + startYesterdayAbsolute := time.Date(now.Year(), now.Month(), now.Day()-days-1, 0, 0, 0, 0, location).In(time.UTC) endYesterdayAbsolute := startYesterdayAbsolute.Add(24 * time.Hour) logger.Info("Running SPKS billing with such timeranges: ", "startOfToday", startOfToday, "startYesterdayAbsolute", startYesterdayAbsolute.Local(), "endYesterdayAbsolute", endYesterdayAbsolute.Local()) - return + odooClient := odoo.NewOdooAPIClient(c, odooURL, odooOauthTokenURL, odooClientId, odooClientSecret, logger, allMetrics["odooMetrics"]) mariadbStandard, mariadbPremium, redisStandard, redisPremium, err := getDatabasesCounts(prometheusURL, prometheusQueryArr, logger, startOfToday, allMetrics)