diff --git a/cmd/bb_replicator/BUILD.bazel b/cmd/bb_replicator/BUILD.bazel index bf000119..fc590b0f 100644 --- a/cmd/bb_replicator/BUILD.bazel +++ b/cmd/bb_replicator/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "//pkg/blobstore/configuration:go_default_library", "//pkg/blobstore/mirrored:go_default_library", "//pkg/digest:go_default_library", + "//pkg/global:go_default_library", "//pkg/grpc:go_default_library", "//pkg/proto/configuration/bb_replicator:go_default_library", "//pkg/proto/replicator:go_default_library", diff --git a/cmd/bb_replicator/main.go b/cmd/bb_replicator/main.go index 0cc6bd75..59496b5a 100644 --- a/cmd/bb_replicator/main.go +++ b/cmd/bb_replicator/main.go @@ -8,6 +8,7 @@ import ( blobstore_configuration "github.com/buildbarn/bb-storage/pkg/blobstore/configuration" "github.com/buildbarn/bb-storage/pkg/blobstore/mirrored" "github.com/buildbarn/bb-storage/pkg/digest" + "github.com/buildbarn/bb-storage/pkg/global" bb_grpc "github.com/buildbarn/bb-storage/pkg/grpc" "github.com/buildbarn/bb-storage/pkg/proto/configuration/bb_replicator" replicator_pb "github.com/buildbarn/bb-storage/pkg/proto/replicator" @@ -25,6 +26,9 @@ func main() { if err := util.UnmarshalConfigurationFromFile(os.Args[1], &configuration); err != nil { log.Fatalf("Failed to read configuration from %s: %s", os.Args[1], err) } + if err := global.ApplyConfiguration(configuration.Global); err != nil { + log.Fatal("Failed to apply global configuration options: ", err) + } source, err := blobstore_configuration.CreateCASBlobAccessObjectFromConfig( configuration.Source, diff --git a/cmd/bb_storage/BUILD.bazel b/cmd/bb_storage/BUILD.bazel index a35134de..130c22fd 100644 --- a/cmd/bb_storage/BUILD.bazel +++ b/cmd/bb_storage/BUILD.bazel @@ -12,8 +12,8 @@ go_library( "//pkg/blobstore/configuration:go_default_library", "//pkg/builder:go_default_library", "//pkg/cas:go_default_library", + "//pkg/global:go_default_library", "//pkg/grpc:go_default_library", - "//pkg/opencensus:go_default_library", "//pkg/proto/configuration/bb_storage:go_default_library", "//pkg/util:go_default_library", "@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:go_default_library", diff --git a/cmd/bb_storage/main.go b/cmd/bb_storage/main.go index d0dc56e8..76b86b79 100644 --- a/cmd/bb_storage/main.go +++ b/cmd/bb_storage/main.go @@ -10,8 +10,8 @@ import ( blobstore_configuration "github.com/buildbarn/bb-storage/pkg/blobstore/configuration" "github.com/buildbarn/bb-storage/pkg/builder" "github.com/buildbarn/bb-storage/pkg/cas" + "github.com/buildbarn/bb-storage/pkg/global" bb_grpc "github.com/buildbarn/bb-storage/pkg/grpc" - "github.com/buildbarn/bb-storage/pkg/opencensus" "github.com/buildbarn/bb-storage/pkg/proto/configuration/bb_storage" "github.com/buildbarn/bb-storage/pkg/util" "github.com/gorilla/mux" @@ -30,9 +30,8 @@ func main() { if err := util.UnmarshalConfigurationFromFile(os.Args[1], &configuration); err != nil { log.Fatalf("Failed to read configuration from %s: %s", os.Args[1], err) } - - if configuration.Jaeger != nil { - opencensus.Initialize(configuration.Jaeger) + if err := global.ApplyConfiguration(configuration.Global); err != nil { + log.Fatal("Failed to apply global configuration options: ", err) } // Storage access. diff --git a/pkg/opencensus/BUILD.bazel b/pkg/global/BUILD.bazel similarity index 63% rename from pkg/opencensus/BUILD.bazel rename to pkg/global/BUILD.bazel index 1ce5674f..c0569671 100644 --- a/pkg/opencensus/BUILD.bazel +++ b/pkg/global/BUILD.bazel @@ -2,12 +2,15 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = ["init.go"], - importpath = "github.com/buildbarn/bb-storage/pkg/opencensus", + srcs = ["apply_configuration.go"], + importpath = "github.com/buildbarn/bb-storage/pkg/global", visibility = ["//visibility:public"], deps = [ - "//pkg/proto/configuration/bb_storage:go_default_library", + "//pkg/proto/configuration/global:go_default_library", + "//pkg/util:go_default_library", + "@com_github_golang_protobuf//ptypes:go_default_library_gen", "@com_github_prometheus_client_golang//prometheus:go_default_library", + "@com_github_prometheus_client_golang//prometheus/push:go_default_library", "@io_opencensus_go//plugin/ocgrpc:go_default_library", "@io_opencensus_go//stats/view:go_default_library", "@io_opencensus_go//trace:go_default_library", diff --git a/pkg/global/apply_configuration.go b/pkg/global/apply_configuration.go new file mode 100644 index 00000000..1d883214 --- /dev/null +++ b/pkg/global/apply_configuration.go @@ -0,0 +1,86 @@ +package global + +import ( + "log" + "runtime" + "time" + + pb "github.com/buildbarn/bb-storage/pkg/proto/configuration/global" + "github.com/buildbarn/bb-storage/pkg/util" + "github.com/golang/protobuf/ptypes" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/push" + + "contrib.go.opencensus.io/exporter/jaeger" + prometheus_exporter "contrib.go.opencensus.io/exporter/prometheus" + "go.opencensus.io/plugin/ocgrpc" + "go.opencensus.io/stats/view" + "go.opencensus.io/trace" + "go.opencensus.io/zpages" +) + +// ApplyConfiguration applies configuration options to the running +// process. These configuration options are global, in that they apply +// to all Buildbarn binaries, regardless of their purpose. +func ApplyConfiguration(configuration *pb.Configuration) error { + // Push traces to Jaeger. + if jaegerConfiguration := configuration.GetJaeger(); jaegerConfiguration != nil { + pe, err := prometheus_exporter.NewExporter(prometheus_exporter.Options{ + Registry: prometheus.DefaultRegisterer.(*prometheus.Registry), + Namespace: "bb_storage", + }) + if err != nil { + return util.StatusWrap(err, "Failed to create the Prometheus stats exporter") + } + view.RegisterExporter(pe) + if err := view.Register(ocgrpc.DefaultServerViews...); err != nil { + return util.StatusWrap(err, "Failed to register ocgrpc server views") + } + zpages.Handle(nil, "/debug") + je, err := jaeger.NewExporter(jaeger.Options{ + AgentEndpoint: jaegerConfiguration.AgentEndpoint, + CollectorEndpoint: jaegerConfiguration.CollectorEndpoint, + Process: jaeger.Process{ + ServiceName: jaegerConfiguration.ServiceName, + }, + }) + if err != nil { + return util.StatusWrap(err, "Failed to create the Jaeger exporter") + } + trace.RegisterExporter(je) + if jaegerConfiguration.AlwaysSample { + trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) + } + } + + // Enable mutex profiling. + runtime.SetMutexProfileFraction(int(configuration.GetMutexProfileFraction())) + + // Periodically push metrics to a Prometheus Pushgateway, as + // opposed to letting the Prometheus server scrape the metrics. + if pushgateway := configuration.GetPrometheusPushgateway(); pushgateway != nil { + pusher := push.New(pushgateway.Url, pushgateway.Job) + pusher.Gatherer(prometheus.DefaultGatherer) + if basicAuthentication := pushgateway.BasicAuthentication; basicAuthentication != nil { + pusher.BasicAuth(basicAuthentication.Username, basicAuthentication.Password) + } + for key, value := range pushgateway.Grouping { + pusher.Grouping(key, value) + } + pushInterval, err := ptypes.Duration(pushgateway.PushInterval) + if err != nil { + return util.StatusWrap(err, "Failed to parse push interval") + } + + go func() { + for { + if err := pusher.Push(); err != nil { + log.Print("Failed to push metrics to Prometheus Pushgateway: ", err) + } + time.Sleep(pushInterval) + } + }() + } + + return nil +} diff --git a/pkg/opencensus/init.go b/pkg/opencensus/init.go deleted file mode 100644 index a86191b7..00000000 --- a/pkg/opencensus/init.go +++ /dev/null @@ -1,46 +0,0 @@ -package opencensus - -import ( - "log" - - "contrib.go.opencensus.io/exporter/jaeger" - prometheus_exporter "contrib.go.opencensus.io/exporter/prometheus" - pb "github.com/buildbarn/bb-storage/pkg/proto/configuration/bb_storage" - "github.com/prometheus/client_golang/prometheus" - "go.opencensus.io/plugin/ocgrpc" - "go.opencensus.io/stats/view" - "go.opencensus.io/trace" - "go.opencensus.io/zpages" -) - -// Initialize sets up Opentracing with Jaeger and a Prometheus exporter. -func Initialize(configuration *pb.JaegerConfiguration) { - if configuration != nil { - pe, err := prometheus_exporter.NewExporter(prometheus_exporter.Options{ - Registry: prometheus.DefaultRegisterer.(*prometheus.Registry), - Namespace: "bb_storage", - }) - if err != nil { - log.Fatalf("Failed to create the Prometheus stats exporter: %v", err) - } - view.RegisterExporter(pe) - if err := view.Register(ocgrpc.DefaultServerViews...); err != nil { - log.Fatalf("Failed to register ocgrpc server views: %v", err) - } - zpages.Handle(nil, "/debug") - je, err := jaeger.NewExporter(jaeger.Options{ - AgentEndpoint: configuration.AgentEndpoint, - CollectorEndpoint: configuration.CollectorEndpoint, - Process: jaeger.Process{ - ServiceName: configuration.ServiceName, - }, - }) - if err != nil { - log.Fatal("Failed to create the Jaeger exporter:", err) - } - trace.RegisterExporter(je) - if configuration.AlwaysSample { - trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) - } - } -} diff --git a/pkg/proto/configuration/bb_replicator/BUILD.bazel b/pkg/proto/configuration/bb_replicator/BUILD.bazel index 11c7613d..5b2c0a03 100644 --- a/pkg/proto/configuration/bb_replicator/BUILD.bazel +++ b/pkg/proto/configuration/bb_replicator/BUILD.bazel @@ -15,6 +15,7 @@ proto_library( visibility = ["//visibility:public"], deps = [ "//pkg/proto/configuration/blobstore:blobstore_proto", + "//pkg/proto/configuration/global:global_proto", "//pkg/proto/configuration/grpc:grpc_proto", ], ) @@ -26,6 +27,7 @@ go_proto_library( visibility = ["//visibility:public"], deps = [ "//pkg/proto/configuration/blobstore:go_default_library", + "//pkg/proto/configuration/global:go_default_library", "//pkg/proto/configuration/grpc:go_default_library", ], ) diff --git a/pkg/proto/configuration/bb_replicator/bb_replicator.proto b/pkg/proto/configuration/bb_replicator/bb_replicator.proto index d69ea56e..04db5d14 100644 --- a/pkg/proto/configuration/bb_replicator/bb_replicator.proto +++ b/pkg/proto/configuration/bb_replicator/bb_replicator.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package buildbarn.configuration.bb_replicator; import "pkg/proto/configuration/blobstore/blobstore.proto"; +import "pkg/proto/configuration/global/global.proto"; import "pkg/proto/configuration/grpc/grpc.proto"; option go_package = "github.com/buildbarn/bb-storage/pkg/proto/configuration/bb_replicator"; @@ -25,4 +26,7 @@ message ApplicationConfiguration { // Maximum Protobuf message size to unmarshal. int64 maximum_message_size_bytes = 6; + + // Common configuration options that apply to all Buildbarn binaries. + buildbarn.configuration.global.Configuration global = 7; } diff --git a/pkg/proto/configuration/bb_storage/BUILD.bazel b/pkg/proto/configuration/bb_storage/BUILD.bazel index 9ffced1a..e0910bb7 100644 --- a/pkg/proto/configuration/bb_storage/BUILD.bazel +++ b/pkg/proto/configuration/bb_storage/BUILD.bazel @@ -8,18 +8,19 @@ proto_library( visibility = ["//visibility:public"], deps = [ "//pkg/proto/configuration/blobstore:blobstore_proto", + "//pkg/proto/configuration/global:global_proto", "//pkg/proto/configuration/grpc:grpc_proto", ], ) go_proto_library( name = "bb_storage_go_proto", - compilers = ["@io_bazel_rules_go//proto:go_grpc"], importpath = "github.com/buildbarn/bb-storage/pkg/proto/configuration/bb_storage", proto = ":bb_storage_proto", visibility = ["//visibility:public"], deps = [ "//pkg/proto/configuration/blobstore:go_default_library", + "//pkg/proto/configuration/global:go_default_library", "//pkg/proto/configuration/grpc:go_default_library", ], ) diff --git a/pkg/proto/configuration/bb_storage/bb_storage.proto b/pkg/proto/configuration/bb_storage/bb_storage.proto index 68897f3e..ee03580b 100644 --- a/pkg/proto/configuration/bb_storage/bb_storage.proto +++ b/pkg/proto/configuration/bb_storage/bb_storage.proto @@ -3,30 +3,17 @@ syntax = "proto3"; package buildbarn.configuration.bb_storage; import "pkg/proto/configuration/blobstore/blobstore.proto"; +import "pkg/proto/configuration/global/global.proto"; import "pkg/proto/configuration/grpc/grpc.proto"; option go_package = "github.com/buildbarn/bb-storage/pkg/proto/configuration/bb_storage"; -message JaegerConfiguration { - // Jaeger agent endpoint. - string agent_endpoint = 1; - - // Jaeger collector endpoint. - string collector_endpoint = 2; - - // OpenTracing service name. - string service_name = 3; - - // Whether or not all traces should be sampled. - bool always_sample = 4; -} - message ApplicationConfiguration { // Blobstore configuration for the bb-storage instance. buildbarn.configuration.blobstore.BlobstoreConfiguration blobstore = 1; - // Jaeger configuration for tracing. - JaegerConfiguration jaeger = 2; + // Jaeger configuration has moved into 'global'. + reserved 2; // Address on which to listen to expose Prometheus metrics. string http_listen_address = 3; @@ -47,4 +34,7 @@ message ApplicationConfiguration { // Maximum Protobuf message size to unmarshal. int64 maximum_message_size_bytes = 8; + + // Common configuration options that apply to all Buildbarn binaries. + buildbarn.configuration.global.Configuration global = 9; } diff --git a/pkg/proto/configuration/global/BUILD.bazel b/pkg/proto/configuration/global/BUILD.bazel new file mode 100644 index 00000000..4cba72a4 --- /dev/null +++ b/pkg/proto/configuration/global/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") + +go_library( + name = "go_default_library", + embed = [":global_go_proto"], + importpath = "github.com/buildbarn/bb-storage/pkg/proto/configuration/global", + visibility = ["//visibility:public"], +) + +proto_library( + name = "global_proto", + srcs = ["global.proto"], + visibility = ["//visibility:public"], + deps = ["@com_google_protobuf//:duration_proto"], +) + +go_proto_library( + name = "global_go_proto", + importpath = "github.com/buildbarn/bb-storage/pkg/proto/configuration/global", + proto = ":global_proto", + visibility = ["//visibility:public"], +) diff --git a/pkg/proto/configuration/global/global.proto b/pkg/proto/configuration/global/global.proto new file mode 100644 index 00000000..3f4f6e26 --- /dev/null +++ b/pkg/proto/configuration/global/global.proto @@ -0,0 +1,60 @@ +syntax = "proto3"; + +package buildbarn.configuration.global; + +import "google/protobuf/duration.proto"; + +option go_package = "github.com/buildbarn/bb-storage/pkg/proto/configuration/global"; + +message JaegerConfiguration { + // Jaeger agent endpoint. + string agent_endpoint = 1; + + // Jaeger collector endpoint. + string collector_endpoint = 2; + + // OpenTracing service name. + string service_name = 3; + + // Whether or not all traces should be sampled. + bool always_sample = 4; +} + +message BasicAuthenticationConfiguration { + // Username to store in the "Authorization: Basic" header. + string username = 1; + + // Password to store in the "Authorization: Basic" header. + string password = 2; +} + +message PrometheusPushgatewayConfiguration { + // URL of the Prometheus Pushgateway server. Do not include the + // "/metrics/jobs/..." part in the URL. + string url = 1; + + // Name of the job to announce to the Prometheus Pushgateway. + string job = 2; + + // If set, enable the use of HTTP basic authentication. + BasicAuthenticationConfiguration basic_authentication = 3; + + // Label pairs to use as the grouping key. + map grouping = 4; + + // Interval between metrics pushes. + google.protobuf.Duration push_interval = 5; +} + +message Configuration { + // Jaeger configuration for tracing. + JaegerConfiguration jaeger = 1; + + // Sets the runtime.SetMutexProfileFraction(), so that the HTTP debug + // endpoints used by pprof expose mutex profiling information. + int32 mutex_profile_fraction = 2; + + // Periodically push metrics to a Prometheus Pushgateway, as opposed + // to letting the Prometheus server scrape the metrics. + PrometheusPushgatewayConfiguration prometheus_pushgateway = 3; +}