Skip to content

[usage] Use a config file to configure usage component #11028

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 4, 2022
Merged

[usage] Use a config file to configure usage component #11028

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions components/usage/BUILD.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,18 @@ packages:
packaging: app
buildCommand: ["go", "build", "-trimpath", "-ldflags", "-buildid= -w -s -X 'github.com/gitpod-io/gitpod/usage/cmd.Version=commit-${__git_commit}'"]

- name: lib
type: go
deps:
- components/common-go:lib
srcs:
- "**/*.go"
- "go.mod"
- "go.sum"
config:
packaging: library
dontTest: true

- name: init-testdb
type: generic
deps:
Expand Down
44 changes: 33 additions & 11 deletions components/usage/cmd/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,14 @@
package cmd

import (
"github.com/gitpod-io/gitpod/usage/pkg/server"
"time"

"bytes"
"encoding/json"
"fmt"
"github.com/gitpod-io/gitpod/common-go/log"
"github.com/gitpod-io/gitpod/usage/pkg/server"
"github.com/spf13/cobra"
"os"
"path"
)

func init() {
Expand All @@ -19,8 +22,7 @@ func init() {
func run() *cobra.Command {
var (
verbose bool
apiKeyFile string
schedule time.Duration
configPath string
)

cmd := &cobra.Command{
Expand All @@ -30,19 +32,39 @@ func run() *cobra.Command {
Run: func(cmd *cobra.Command, args []string) {
log.Init(ServiceName, Version, true, verbose)

err := server.Start(server.Config{
ControllerSchedule: schedule,
StripeCredentialsFile: apiKeyFile,
})
cfg, err := parseConfig(configPath)
if err != nil {
log.WithError(err).Fatal("Failed to get config. Did you specify --config correctly?")
}

err = server.Start(cfg)
if err != nil {
log.WithError(err).Fatal("Failed to start usage server.")
}
},
}

localConfig := path.Join(os.ExpandEnv("GOMOD"), "..", "config.json")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the use case here? Why do we expect config.json one level up from the go.mod?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

GOMOD points to where the go.mod is defined, in our case that is GOMOD="/workspace/gitpod/components/usage/go.mod". The .. drops the filename and we then join it with our desired filename.

See playground


cmd.Flags().BoolVar(&verbose, "verbose", false, "Toggle verbose logging (debug level)")
cmd.Flags().DurationVar(&schedule, "schedule", 1*time.Hour, "The schedule on which the reconciler should run")
cmd.Flags().StringVar(&apiKeyFile, "stripe-secret-path", "", "Location of the Stripe credentials file on disk")
cmd.Flags().StringVar(&configPath, "config", localConfig, "Configuration file for running usage component")

return cmd
}

func parseConfig(path string) (server.Config, error) {
raw, err := os.ReadFile(path)
if err != nil {
return server.Config{}, fmt.Errorf("failed to read config from %s: %w", path, err)
}

var cfg server.Config
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
err = dec.Decode(&cfg)
if err != nil {
return server.Config{}, fmt.Errorf("failed to parse config from %s: %w", path, err)
}

return cfg, nil
}
3 changes: 3 additions & 0 deletions components/usage/config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"controllerSchedule": "1h"
}
94 changes: 47 additions & 47 deletions components/usage/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,53 @@ module github.com/gitpod-io/gitpod/usage

go 1.18

require (
github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000
github.com/go-sql-driver/mysql v1.6.0
github.com/google/uuid v1.1.2
github.com/prometheus/client_golang v1.12.1
github.com/relvacode/iso8601 v1.1.0
github.com/robfig/cron v1.2.0
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.4.0
github.com/stretchr/testify v1.7.0
github.com/stripe/stripe-go/v72 v72.114.0
gorm.io/datatypes v1.0.6
gorm.io/driver/mysql v1.3.3
gorm.io/gorm v1.23.5
)

require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/hashicorp/golang-lru v0.5.1 // indirect
github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.4 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154 // indirect
google.golang.org/grpc v1.45.0 // indirect
google.golang.org/protobuf v1.28.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)

replace github.com/gitpod-io/gitpod/common-go => ../common-go // leeway

replace k8s.io/api => k8s.io/api v0.23.5 // leeway indirect from components/common-go:lib
Expand Down Expand Up @@ -53,50 +100,3 @@ replace k8s.io/kubectl => k8s.io/kubectl v0.23.5 // leeway indirect from compone
replace k8s.io/mount-utils => k8s.io/mount-utils v0.23.5 // leeway indirect from components/common-go:lib

replace k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.23.5 // leeway indirect from components/common-go:lib

require (
github.com/gitpod-io/gitpod/common-go v0.0.0-00010101000000-000000000000
github.com/go-sql-driver/mysql v1.6.0
github.com/google/uuid v1.1.2
github.com/prometheus/client_golang v1.12.1
github.com/relvacode/iso8601 v1.1.0
github.com/robfig/cron v1.2.0
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.4.0
github.com/stretchr/testify v1.7.0
github.com/stripe/stripe-go/v72 v72.114.0
gorm.io/datatypes v1.0.6
gorm.io/driver/mysql v1.3.3
gorm.io/gorm v1.23.5
)

require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/hashicorp/golang-lru v0.5.1 // indirect
github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.4 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154 // indirect
google.golang.org/grpc v1.45.0 // indirect
google.golang.org/protobuf v1.28.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)
9 changes: 7 additions & 2 deletions components/usage/pkg/server/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (

type Config struct {
// ControllerSchedule determines how frequently to run the Usage/Billing controller
ControllerSchedule time.Duration `json:"controllerSchedule,omitempty"`
ControllerSchedule string `json:"controllerSchedule,omitempty"`

StripeCredentialsFile string `json:"stripeCredentialsFile,omitempty"`

Expand Down Expand Up @@ -53,7 +53,12 @@ func Start(cfg Config) error {
billingController = controller.NewStripeBillingController(c, controller.DefaultWorkspacePricer)
}

ctrl, err := controller.New(cfg.ControllerSchedule, controller.NewUsageReconciler(conn, billingController))
schedule, err := time.ParseDuration(cfg.ControllerSchedule)
if err != nil {
return fmt.Errorf("failed to parse schedule duration: %w", err)
}

ctrl, err := controller.New(schedule, controller.NewUsageReconciler(conn, billingController))
if err != nil {
return fmt.Errorf("failed to initialize usage controller: %w", err)
}
Expand Down
1 change: 1 addition & 0 deletions install/installer/BUILD.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ packages:
- components/ws-daemon:lib
- components/ws-manager-api/go:lib
- components/ws-proxy:lib
- components/usage:lib
env:
- CGO_ENABLED=0
prep:
Expand Down
14 changes: 13 additions & 1 deletion install/installer/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ require (
github.com/gitpod-io/gitpod/openvsx-proxy v0.0.0-00010101000000-000000000000
github.com/gitpod-io/gitpod/public-api v0.0.0-00010101000000-000000000000
github.com/gitpod-io/gitpod/registry-facade/api v0.0.0-00010101000000-000000000000
github.com/gitpod-io/gitpod/usage v0.0.0-00010101000000-000000000000
github.com/gitpod-io/gitpod/ws-daemon v0.0.0-00010101000000-000000000000
github.com/gitpod-io/gitpod/ws-daemon/api v0.0.0-00010101000000-000000000000
github.com/gitpod-io/gitpod/ws-manager/api v0.0.0-00010101000000-000000000000
Expand All @@ -23,7 +24,7 @@ require (
github.com/jetstack/cert-manager v1.4.4
github.com/replicatedhq/kots v1.67.0
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.2.1
github.com/spf13/cobra v1.4.0
github.com/stretchr/testify v1.7.1
golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898
helm.sh/helm/v3 v3.7.1
Expand Down Expand Up @@ -111,6 +112,7 @@ require (
github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/go-redis/redis/v7 v7.4.1 // indirect
github.com/go-redis/redis/v8 v8.11.5 // indirect
github.com/go-sql-driver/mysql v1.6.0 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/godbus/dbus/v5 v5.0.6 // indirect
github.com/gogo/googleapis v1.4.0 // indirect
Expand Down Expand Up @@ -163,6 +165,8 @@ require (
github.com/ipld/go-codec-dagpb v1.3.0 // indirect
github.com/ipld/go-ipld-prime v0.11.0 // indirect
github.com/jbenet/goprocess v0.1.4 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.4 // indirect
github.com/jmoiron/sqlx v1.3.1 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
Expand Down Expand Up @@ -223,7 +227,9 @@ require (
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/relvacode/iso8601 v1.1.0 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/robfig/cron v1.2.0 // indirect
github.com/rs/cors v1.7.0 // indirect
github.com/rs/xid v1.2.1 // indirect
github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc // indirect
Expand All @@ -233,6 +239,7 @@ require (
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stripe/stripe-go/v72 v72.114.0 // indirect
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
github.com/uber/jaeger-client-go v2.29.1+incompatible // indirect
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
Expand Down Expand Up @@ -267,6 +274,9 @@ require (
gopkg.in/ini.v1 v1.63.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
gorm.io/datatypes v1.0.6 // indirect
gorm.io/driver/mysql v1.3.3 // indirect
gorm.io/gorm v1.23.5 // indirect
k8s.io/apiextensions-apiserver v0.23.5 // indirect
k8s.io/apiserver v0.23.5 // indirect
k8s.io/cli-runtime v0.23.5 // indirect
Expand Down Expand Up @@ -309,6 +319,8 @@ replace github.com/gitpod-io/gitpod/registry-facade/api => ../../components/regi

replace github.com/gitpod-io/gitpod/supervisor/api => ../../components/supervisor-api/go // leeway

replace github.com/gitpod-io/gitpod/usage => ../../components/usage // leeway

replace github.com/gitpod-io/gitpod/ws-daemon => ../../components/ws-daemon // leeway

replace github.com/gitpod-io/gitpod/ws-daemon/api => ../../components/ws-daemon-api/go // leeway
Expand Down
Loading