From 1ba5acb583e99d9009df8d6f07840d2736a6bab7 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 5 Sep 2023 10:25:18 +0200 Subject: [PATCH] add clientlog service Signed-off-by: jkoberg --- .drone.star | 1 + Makefile | 1 + changelog/unreleased/clientlog-service.md | 5 + .../general-info/new-service-checklist.md | 2 + ocis-pkg/config/config.go | 2 + ocis-pkg/config/defaultconfig.go | 2 + ocis/pkg/command/services.go | 6 + ocis/pkg/init/init.go | 8 + ocis/pkg/runtime/service/service.go | 6 + services/clientlog/Makefile | 38 +++++ services/clientlog/README.md | 14 ++ services/clientlog/cmd/clientlog/main.go | 14 ++ services/clientlog/pkg/command/health.go | 18 +++ services/clientlog/pkg/command/root.go | 34 +++++ services/clientlog/pkg/command/server.go | 137 ++++++++++++++++++ services/clientlog/pkg/command/version.go | 19 +++ services/clientlog/pkg/config/config.go | 49 +++++++ services/clientlog/pkg/config/debug.go | 9 ++ .../pkg/config/defaults/defaultconfig.go | 80 ++++++++++ services/clientlog/pkg/config/log.go | 9 ++ services/clientlog/pkg/config/parser/parse.go | 43 ++++++ services/clientlog/pkg/config/service.go | 6 + services/clientlog/pkg/config/tracing.go | 21 +++ services/clientlog/pkg/logging/logging.go | 17 +++ services/clientlog/pkg/metrics/metrics.go | 35 +++++ services/clientlog/pkg/service/options.go | 65 +++++++++ services/clientlog/pkg/service/service.go | 134 +++++++++++++++++ .../pkg/service/service_suit_test.go | 26 ++++ .../clientlog/pkg/service/service_test.go | 3 + services/sse/README.md | 7 + services/userlog/README.md | 7 + 31 files changed, 818 insertions(+) create mode 100644 changelog/unreleased/clientlog-service.md create mode 100644 services/clientlog/Makefile create mode 100644 services/clientlog/README.md create mode 100644 services/clientlog/cmd/clientlog/main.go create mode 100644 services/clientlog/pkg/command/health.go create mode 100644 services/clientlog/pkg/command/root.go create mode 100644 services/clientlog/pkg/command/server.go create mode 100644 services/clientlog/pkg/command/version.go create mode 100644 services/clientlog/pkg/config/config.go create mode 100644 services/clientlog/pkg/config/debug.go create mode 100644 services/clientlog/pkg/config/defaults/defaultconfig.go create mode 100644 services/clientlog/pkg/config/log.go create mode 100644 services/clientlog/pkg/config/parser/parse.go create mode 100644 services/clientlog/pkg/config/service.go create mode 100644 services/clientlog/pkg/config/tracing.go create mode 100644 services/clientlog/pkg/logging/logging.go create mode 100644 services/clientlog/pkg/metrics/metrics.go create mode 100644 services/clientlog/pkg/service/options.go create mode 100644 services/clientlog/pkg/service/service.go create mode 100644 services/clientlog/pkg/service/service_suit_test.go create mode 100644 services/clientlog/pkg/service/service_test.go diff --git a/.drone.star b/.drone.star index 63fdd685239..4d72ae6b4d4 100644 --- a/.drone.star +++ b/.drone.star @@ -62,6 +62,7 @@ config = { "services/auth-bearer", "services/auth-machine", "services/auth-service", + "services/clientlog", "services/eventhistory", "services/frontend", "services/gateway", diff --git a/Makefile b/Makefile index f45ce7f1758..b05218ba13f 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ OCIS_MODULES = \ services/auth-bearer \ services/auth-machine \ services/auth-service \ + services/clientlog \ services/eventhistory \ services/frontend \ services/gateway \ diff --git a/changelog/unreleased/clientlog-service.md b/changelog/unreleased/clientlog-service.md new file mode 100644 index 00000000000..e8b2f111bd8 --- /dev/null +++ b/changelog/unreleased/clientlog-service.md @@ -0,0 +1,5 @@ +Enhancement: Introduce clientlog service + +Add the clientlog service which will send machine readable notifications to clients + +https://github.com/owncloud/ocis/pull/7217 diff --git a/docs/services/general-info/new-service-checklist.md b/docs/services/general-info/new-service-checklist.md index 45dbf364bd8..dd50d3f1e7f 100644 --- a/docs/services/general-info/new-service-checklist.md +++ b/docs/services/general-info/new-service-checklist.md @@ -28,6 +28,8 @@ Use this checklist with copy/paste in your PR - right from the beginning. It ren - [ ] Make the service startable for binary and individual startup: - For single binary add service to `ocis/pkg/runtime` - For individual startup add service to `ocis/pkg/commands` + - Add the service config to `ocis-pkg/config/defaultconfig.go` +- [ ] If the service is using service accounts, add it to `ocis/pkg/init/init.go` - [ ] Add the service to `.drone.star` to enable CI. - [ ] Inform doc team in an _early stage_ to review the readme AND the environment variables created. - The description must reflect the behaviour AND usually has a positive code quality impact. diff --git a/ocis-pkg/config/config.go b/ocis-pkg/config/config.go index 77fec4f6fa8..8473d3877b7 100644 --- a/ocis-pkg/config/config.go +++ b/ocis-pkg/config/config.go @@ -10,6 +10,7 @@ import ( authbearer "github.com/owncloud/ocis/v2/services/auth-bearer/pkg/config" authmachine "github.com/owncloud/ocis/v2/services/auth-machine/pkg/config" authservice "github.com/owncloud/ocis/v2/services/auth-service/pkg/config" + clientlog "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" eventhistory "github.com/owncloud/ocis/v2/services/eventhistory/pkg/config" frontend "github.com/owncloud/ocis/v2/services/frontend/pkg/config" gateway "github.com/owncloud/ocis/v2/services/gateway/pkg/config" @@ -85,6 +86,7 @@ type Config struct { AuthBearer *authbearer.Config `yaml:"auth_bearer"` AuthMachine *authmachine.Config `yaml:"auth_machine"` AuthService *authservice.Config `yaml:"auth_service"` + Clientlog *clientlog.Config `yaml:"clientlog"` EventHistory *eventhistory.Config `yaml:"eventhistory"` Frontend *frontend.Config `yaml:"frontend"` Gateway *gateway.Config `yaml:"gateway"` diff --git a/ocis-pkg/config/defaultconfig.go b/ocis-pkg/config/defaultconfig.go index 8639821bc76..3d068b09b93 100644 --- a/ocis-pkg/config/defaultconfig.go +++ b/ocis-pkg/config/defaultconfig.go @@ -9,6 +9,7 @@ import ( authbearer "github.com/owncloud/ocis/v2/services/auth-bearer/pkg/config/defaults" authmachine "github.com/owncloud/ocis/v2/services/auth-machine/pkg/config/defaults" authservice "github.com/owncloud/ocis/v2/services/auth-service/pkg/config/defaults" + clientlog "github.com/owncloud/ocis/v2/services/clientlog/pkg/config/defaults" eventhistory "github.com/owncloud/ocis/v2/services/eventhistory/pkg/config/defaults" frontend "github.com/owncloud/ocis/v2/services/frontend/pkg/config/defaults" gateway "github.com/owncloud/ocis/v2/services/gateway/pkg/config/defaults" @@ -57,6 +58,7 @@ func DefaultConfig() *Config { AuthBearer: authbearer.DefaultConfig(), AuthMachine: authmachine.DefaultConfig(), AuthService: authservice.DefaultConfig(), + Clientlog: clientlog.DefaultConfig(), EventHistory: eventhistory.DefaultConfig(), Frontend: frontend.DefaultConfig(), Gateway: gateway.DefaultConfig(), diff --git a/ocis/pkg/command/services.go b/ocis/pkg/command/services.go index da634e2dfa0..b745f5d2587 100644 --- a/ocis/pkg/command/services.go +++ b/ocis/pkg/command/services.go @@ -16,6 +16,7 @@ import ( authbearer "github.com/owncloud/ocis/v2/services/auth-bearer/pkg/command" authmachine "github.com/owncloud/ocis/v2/services/auth-machine/pkg/command" authservice "github.com/owncloud/ocis/v2/services/auth-service/pkg/command" + clientlog "github.com/owncloud/ocis/v2/services/clientlog/pkg/command" eventhistory "github.com/owncloud/ocis/v2/services/eventhistory/pkg/command" frontend "github.com/owncloud/ocis/v2/services/frontend/pkg/command" gateway "github.com/owncloud/ocis/v2/services/gateway/pkg/command" @@ -89,6 +90,11 @@ var svccmds = []register.Command{ cfg.AuthService.Commons = cfg.Commons }) }, + func(cfg *config.Config) *cli.Command { + return ServiceCommand(cfg, cfg.Clientlog.Service.Name, clientlog.GetCommands(cfg.Clientlog), func(c *config.Config) { + cfg.Clientlog.Commons = cfg.Commons + }) + }, func(cfg *config.Config) *cli.Command { return ServiceCommand(cfg, cfg.EventHistory.Service.Name, eventhistory.GetCommands(cfg.EventHistory), func(c *config.Config) { cfg.EventHistory.Commons = cfg.Commons diff --git a/ocis/pkg/init/init.go b/ocis/pkg/init/init.go index 01aef0ceab1..9ce78d63088 100644 --- a/ocis/pkg/init/init.go +++ b/ocis/pkg/init/init.go @@ -141,6 +141,10 @@ type AuthService struct { ServiceAccount ServiceAccount `yaml:"service_account"` } +type Clientlog struct { + ServiceAccount ServiceAccount `yaml:"service_account"` +} + type Nats struct { // The nats config has a field called nats Nats struct { @@ -193,6 +197,7 @@ type OcisConfig struct { Gateway Gateway Userlog Userlog AuthService AuthService `yaml:"auth_service"` + Clientlog Clientlog } func checkConfigPath(configPath string) error { @@ -377,6 +382,9 @@ func CreateConfig(insecure, forceOverwrite bool, configPath, adminPassword strin Notifications: Notifications{ ServiceAccount: serviceAccount, }, + Clientlog: Clientlog{ + ServiceAccount: serviceAccount, + }, } if insecure { diff --git a/ocis/pkg/runtime/service/service.go b/ocis/pkg/runtime/service/service.go index 1f43db921eb..9687867fd04 100644 --- a/ocis/pkg/runtime/service/service.go +++ b/ocis/pkg/runtime/service/service.go @@ -27,6 +27,7 @@ import ( authbasic "github.com/owncloud/ocis/v2/services/auth-basic/pkg/command" authmachine "github.com/owncloud/ocis/v2/services/auth-machine/pkg/command" authservice "github.com/owncloud/ocis/v2/services/auth-service/pkg/command" + clientlog "github.com/owncloud/ocis/v2/services/clientlog/pkg/command" eventhistory "github.com/owncloud/ocis/v2/services/eventhistory/pkg/command" frontend "github.com/owncloud/ocis/v2/services/frontend/pkg/command" gateway "github.com/owncloud/ocis/v2/services/gateway/pkg/command" @@ -141,6 +142,11 @@ func NewService(options ...Option) (*Service, error) { cfg.AuthService.Commons = cfg.Commons return authservice.Execute(cfg.AuthService) }) + reg(opts.Config.Clientlog.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error { + cfg.Clientlog.Context = ctx + cfg.Clientlog.Commons = cfg.Commons + return clientlog.Execute(cfg.Clientlog) + }) reg(opts.Config.EventHistory.Service.Name, func(ctx context.Context, cfg *ociscfg.Config) error { cfg.EventHistory.Context = ctx cfg.EventHistory.Commons = cfg.Commons diff --git a/services/clientlog/Makefile b/services/clientlog/Makefile new file mode 100644 index 00000000000..5cf6d0b10f3 --- /dev/null +++ b/services/clientlog/Makefile @@ -0,0 +1,38 @@ +SHELL := bash +NAME := clientlog + +include ../../.make/recursion.mk + +############ tooling ############ +ifneq (, $(shell command -v go 2> /dev/null)) # suppress `command not found warnings` for non go targets in CI +include ../../.bingo/Variables.mk +endif + +############ go tooling ############ +include ../../.make/go.mk + +############ release ############ +include ../../.make/release.mk + +############ docs generate ############ +include ../../.make/docs.mk + +.PHONY: docs-generate +docs-generate: config-docs-generate + +############ generate ############ +include ../../.make/generate.mk + +.PHONY: ci-go-generate +ci-go-generate: $(MOCKERY) # CI runs ci-node-generate automatically before this target + $(MOCKERY) --dir ../../protogen/gen/ocis/services/eventhistory/v0 --case underscore --name EventHistoryService + +.PHONY: ci-node-generate +ci-node-generate: + +############ licenses ############ +.PHONY: ci-node-check-licenses +ci-node-check-licenses: + +.PHONY: ci-node-save-licenses +ci-node-save-licenses: diff --git a/services/clientlog/README.md b/services/clientlog/README.md new file mode 100644 index 00000000000..9421f5b3bd4 --- /dev/null +++ b/services/clientlog/README.md @@ -0,0 +1,14 @@ +# Clientlog service + +The `clientlog` service is responsible for composing machine readable notifications for clients + +## The `...log` service ecosystem + +`...log` services (`userlog`, `clientlog`) are responsible for composing notifications for a certain audience. + - `userlog` service translates and adjust messages to be human readable + - `clientlog` service composes machine readable messages so clients can act without needing to query the server + - `sse` service is only responsible for sending these messages. It does not care about their form or language + +## `clientlog` events + +The messages the `clientlog` service sends are meant to be used by clients, not by users. The client might for example be informed that a file is finished postprocessing, so it can make the file available to the user without needing to make another call to the server. diff --git a/services/clientlog/cmd/clientlog/main.go b/services/clientlog/cmd/clientlog/main.go new file mode 100644 index 00000000000..ce0d4a4aff9 --- /dev/null +++ b/services/clientlog/cmd/clientlog/main.go @@ -0,0 +1,14 @@ +package main + +import ( + "os" + + "github.com/owncloud/ocis/v2/services/clientlog/pkg/command" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config/defaults" +) + +func main() { + if err := command.Execute(defaults.DefaultConfig()); err != nil { + os.Exit(1) + } +} diff --git a/services/clientlog/pkg/command/health.go b/services/clientlog/pkg/command/health.go new file mode 100644 index 00000000000..69e2b10ec21 --- /dev/null +++ b/services/clientlog/pkg/command/health.go @@ -0,0 +1,18 @@ +package command + +import ( + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" + "github.com/urfave/cli/v2" +) + +// Health is the entrypoint for the health command. +func Health(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "health", + Usage: "Check health status", + Action: func(c *cli.Context) error { + // Not implemented + return nil + }, + } +} diff --git a/services/clientlog/pkg/command/root.go b/services/clientlog/pkg/command/root.go new file mode 100644 index 00000000000..389caf1bb21 --- /dev/null +++ b/services/clientlog/pkg/command/root.go @@ -0,0 +1,34 @@ +package command + +import ( + "os" + + "github.com/owncloud/ocis/v2/ocis-pkg/clihelper" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" + "github.com/urfave/cli/v2" +) + +// GetCommands provides all commands for this service +func GetCommands(cfg *config.Config) cli.Commands { + return []*cli.Command{ + // start this service + Server(cfg), + + // interaction with this service + + // infos about this service + Health(cfg), + Version(cfg), + } +} + +// Execute is the entry point for the clientlog command. +func Execute(cfg *config.Config) error { + app := clihelper.DefaultApp(&cli.App{ + Name: "clientlog", + Usage: "starts clientlog service", + Commands: GetCommands(cfg), + }) + + return app.Run(os.Args) +} diff --git a/services/clientlog/pkg/command/server.go b/services/clientlog/pkg/command/server.go new file mode 100644 index 00000000000..b149a6ce27d --- /dev/null +++ b/services/clientlog/pkg/command/server.go @@ -0,0 +1,137 @@ +package command + +import ( + "context" + "fmt" + + "github.com/cs3org/reva/v2/pkg/events" + "github.com/cs3org/reva/v2/pkg/events/stream" + "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" + "github.com/oklog/run" + "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" + "github.com/owncloud/ocis/v2/ocis-pkg/handlers" + "github.com/owncloud/ocis/v2/ocis-pkg/registry" + "github.com/owncloud/ocis/v2/ocis-pkg/service/debug" + "github.com/owncloud/ocis/v2/ocis-pkg/tracing" + "github.com/owncloud/ocis/v2/ocis-pkg/version" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config/parser" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/logging" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/metrics" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/service" + "github.com/urfave/cli/v2" +) + +// all events we care about +var _registeredEvents = []events.Unmarshaller{ + events.UploadReady{}, +} + +// Server is the entrypoint for the server command. +func Server(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "server", + Usage: fmt.Sprintf("start the %s service without runtime (unsupervised mode)", cfg.Service.Name), + Category: "server", + Before: func(c *cli.Context) error { + return configlog.ReturnFatal(parser.ParseConfig(cfg)) + }, + Action: func(c *cli.Context) error { + logger := logging.Configure(cfg.Service.Name, cfg.Log) + tracerProvider, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name) + if err != nil { + return err + } + + /* + grpcClient, err := ogrpc.NewClient( + append(ogrpc.GetClientOptions(cfg.GRPCClientTLS), ogrpc.WithTraceProvider(tracerProvider))..., + ) + if err != nil { + return err + } + */ // TODO: remove + + gr := run.Group{} + ctx, cancel := func() (context.Context, context.CancelFunc) { + if cfg.Context == nil { + return context.WithCancel(context.Background()) + } + return context.WithCancel(cfg.Context) + }() + + mtrcs := metrics.New() + mtrcs.BuildInfo.WithLabelValues(version.GetString()).Set(1) + + defer cancel() + + stream, err := stream.NatsFromConfig(cfg.Service.Name, stream.NatsConfig(cfg.Events)) + if err != nil { + return err + } + + tm, err := pool.StringToTLSMode(cfg.GRPCClientTLS.Mode) + if err != nil { + return err + } + gatewaySelector, err := pool.GatewaySelector( + cfg.RevaGateway, + pool.WithTLSCACert(cfg.GRPCClientTLS.CACert), + pool.WithTLSMode(tm), + pool.WithRegistry(registry.GetRegistry()), + pool.WithTracerProvider(tracerProvider), + ) + if err != nil { + return fmt.Errorf("could not get reva client selector: %s", err) + } + + { + svc, err := service.NewClientlogService( + service.Logger(logger), + service.Config(cfg), + service.Stream(stream), + service.GatewaySelector(gatewaySelector), + service.RegisteredEvents(_registeredEvents), + service.TraceProvider(tracerProvider), + ) + + if err != nil { + logger.Info().Err(err).Str("transport", "http").Msg("Failed to initialize server") + return err + } + + gr.Add(func() error { + return svc.Run() + }, func(err error) { + logger.Error(). + Str("transport", "http"). + Err(err). + Msg("Shutting down server") + + cancel() + }) + } + + { + server := debug.NewService( + debug.Logger(logger), + debug.Name(cfg.Service.Name), + debug.Version(version.GetString()), + debug.Address(cfg.Debug.Addr), + debug.Token(cfg.Debug.Token), + debug.Pprof(cfg.Debug.Pprof), + debug.Zpages(cfg.Debug.Zpages), + debug.Health(handlers.Health), + debug.Ready(handlers.Ready), + ) + + gr.Add(server.ListenAndServe, func(_ error) { + _ = server.Shutdown(ctx) + cancel() + }) + } + + return gr.Run() + }, + } +} diff --git a/services/clientlog/pkg/command/version.go b/services/clientlog/pkg/command/version.go new file mode 100644 index 00000000000..73c8776baab --- /dev/null +++ b/services/clientlog/pkg/command/version.go @@ -0,0 +1,19 @@ +package command + +import ( + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" + "github.com/urfave/cli/v2" +) + +// Version prints the service versions of all running instances. +func Version(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "version", + Usage: "print the version of this binary and the running service instances", + Category: "info", + Action: func(c *cli.Context) error { + // not implemented + return nil + }, + } +} diff --git a/services/clientlog/pkg/config/config.go b/services/clientlog/pkg/config/config.go new file mode 100644 index 00000000000..929950da905 --- /dev/null +++ b/services/clientlog/pkg/config/config.go @@ -0,0 +1,49 @@ +package config + +import ( + "context" + + "github.com/owncloud/ocis/v2/ocis-pkg/shared" +) + +// Config combines all available configuration parts. +type Config struct { + Commons *shared.Commons `yaml:"-"` // don't use this directly as configuration for a service + + Service Service `yaml:"-"` + + Tracing *Tracing `yaml:"tracing"` + Log *Log `yaml:"log"` + Debug Debug `yaml:"debug"` + + GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"` + + TokenManager *TokenManager `yaml:"token_manager"` + + RevaGateway string `yaml:"reva_gateway" env:"OCIS_REVA_GATEWAY" desc:"CS3 gateway used to look up user metadata"` + Events Events `yaml:"events"` + + ServiceAccount ServiceAccount `yaml:"service_account"` + + Context context.Context `yaml:"-"` +} + +// Events combines the configuration options for the event bus. +type Events struct { + Endpoint string `yaml:"endpoint" env:"OCIS_EVENTS_ENDPOINT;CLIENTLOG_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture."` + Cluster string `yaml:"cluster" env:"OCIS_EVENTS_CLUSTER;CLIENTLOG_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system."` + TLSInsecure bool `yaml:"tls_insecure" env:"OCIS_INSECURE;CLIENTLOG_EVENTS_TLS_INSECURE" desc:"Whether to verify the server TLS certificates."` + TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"OCIS_EVENTS_TLS_ROOT_CA_CERTIFICATE;CLIENTLOG_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided NOTIFICATIONS_EVENTS_TLS_INSECURE will be seen as false."` + EnableTLS bool `yaml:"enable_tls" env:"OCIS_EVENTS_ENABLE_TLS;CLIENTLOG_EVENTS_ENABLE_TLS" desc:"Enable TLS for the connection to the events broker. The events broker is the ocis service which receives and delivers events between the services.."` +} + +// TokenManager is the config for using the reva token manager +type TokenManager struct { + JWTSecret string `yaml:"jwt_secret" env:"OCIS_JWT_SECRET;CLIENTLOG_JWT_SECRET" desc:"The secret to mint and validate jwt tokens."` +} + +// ServiceAccount is the configuration for the used service account +type ServiceAccount struct { + ServiceAccountID string `yaml:"service_account_id" env:"OCIS_SERVICE_ACCOUNT_ID;CLIENTLOG_SERVICE_ACCOUNT_ID" desc:"The ID of the service account the service should use. See the 'auth-service' service description for more details."` + ServiceAccountSecret string `yaml:"service_account_secret" env:"OCIS_SERVICE_ACCOUNT_SECRET;CLIENTLOG_SERVICE_ACCOUNT_SECRET" desc:"The service account secret."` +} diff --git a/services/clientlog/pkg/config/debug.go b/services/clientlog/pkg/config/debug.go new file mode 100644 index 00000000000..d29ac49163a --- /dev/null +++ b/services/clientlog/pkg/config/debug.go @@ -0,0 +1,9 @@ +package config + +// Debug defines the available debug configuration. +type Debug struct { + Addr string `yaml:"addr" env:"USERLOG_DEBUG_ADDR" desc:"Bind address of the debug server, where metrics, health, config and debug endpoints will be exposed."` + Token string `yaml:"token" env:"USERLOG_DEBUG_TOKEN" desc:"Token to secure the metrics endpoint."` + Pprof bool `yaml:"pprof" env:"USERLOG_DEBUG_PPROF" desc:"Enables pprof, which can be used for profiling."` + Zpages bool `yaml:"zpages" env:"USERLOG_DEBUG_ZPAGES" desc:"Enables zpages, which can be used for collecting and viewing in-memory traces."` +} diff --git a/services/clientlog/pkg/config/defaults/defaultconfig.go b/services/clientlog/pkg/config/defaults/defaultconfig.go new file mode 100644 index 00000000000..b0a2951bdc1 --- /dev/null +++ b/services/clientlog/pkg/config/defaults/defaultconfig.go @@ -0,0 +1,80 @@ +package defaults + +import ( + "github.com/owncloud/ocis/v2/ocis-pkg/shared" + "github.com/owncloud/ocis/v2/ocis-pkg/structs" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" +) + +// FullDefaultConfig returns the full default config +func FullDefaultConfig() *config.Config { + cfg := DefaultConfig() + EnsureDefaults(cfg) + Sanitize(cfg) + return cfg +} + +// DefaultConfig return the default configuration +func DefaultConfig() *config.Config { + return &config.Config{ + Debug: config.Debug{ + Addr: "127.0.0.1:9260", + Token: "", + Pprof: false, + Zpages: false, + }, + Service: config.Service{ + Name: "clientlog", + }, + Events: config.Events{ + Endpoint: "127.0.0.1:9233", + Cluster: "ocis-cluster", + EnableTLS: false, + }, + RevaGateway: shared.DefaultRevaConfig().Address, + } +} + +// EnsureDefaults ensures the config contains default values +func EnsureDefaults(cfg *config.Config) { + // provide with defaults for shared logging, since we need a valid destination address for "envdecode". + if cfg.Log == nil && cfg.Commons != nil && cfg.Commons.Log != nil { + cfg.Log = &config.Log{ + Level: cfg.Commons.Log.Level, + Pretty: cfg.Commons.Log.Pretty, + Color: cfg.Commons.Log.Color, + File: cfg.Commons.Log.File, + } + } else if cfg.Log == nil { + cfg.Log = &config.Log{} + } + + if cfg.GRPCClientTLS == nil && cfg.Commons != nil { + cfg.GRPCClientTLS = structs.CopyOrZeroValue(cfg.Commons.GRPCClientTLS) + } + + if cfg.TokenManager == nil && cfg.Commons != nil && cfg.Commons.TokenManager != nil { + cfg.TokenManager = &config.TokenManager{ + JWTSecret: cfg.Commons.TokenManager.JWTSecret, + } + } else if cfg.TokenManager == nil { + cfg.TokenManager = &config.TokenManager{} + } + + // provide with defaults for shared tracing, since we need a valid destination address for "envdecode". + if cfg.Tracing == nil && cfg.Commons != nil && cfg.Commons.Tracing != nil { + cfg.Tracing = &config.Tracing{ + Enabled: cfg.Commons.Tracing.Enabled, + Type: cfg.Commons.Tracing.Type, + Endpoint: cfg.Commons.Tracing.Endpoint, + Collector: cfg.Commons.Tracing.Collector, + } + } else if cfg.Tracing == nil { + cfg.Tracing = &config.Tracing{} + } +} + +// Sanitize sanitizes the config +func Sanitize(cfg *config.Config) { + // sanitize config +} diff --git a/services/clientlog/pkg/config/log.go b/services/clientlog/pkg/config/log.go new file mode 100644 index 00000000000..c3cc38911fe --- /dev/null +++ b/services/clientlog/pkg/config/log.go @@ -0,0 +1,9 @@ +package config + +// Log defines the available log configuration. +type Log struct { + Level string `mapstructure:"level" env:"OCIS_LOG_LEVEL;USERLOG_LOG_LEVEL" desc:"The log level. Valid values are: 'panic', 'fatal', 'error', 'warn', 'info', 'debug', 'trace'."` + Pretty bool `mapstructure:"pretty" env:"OCIS_LOG_PRETTY;USERLOG_LOG_PRETTY" desc:"Activates pretty log output."` + Color bool `mapstructure:"color" env:"OCIS_LOG_COLOR;USERLOG_LOG_COLOR" desc:"Activates colorized log output."` + File string `mapstructure:"file" env:"OCIS_LOG_FILE;USERLOG_LOG_FILE" desc:"The path to the log file. Activates logging to this file if set."` +} diff --git a/services/clientlog/pkg/config/parser/parse.go b/services/clientlog/pkg/config/parser/parse.go new file mode 100644 index 00000000000..99ad1d14cc9 --- /dev/null +++ b/services/clientlog/pkg/config/parser/parse.go @@ -0,0 +1,43 @@ +package parser + +import ( + "errors" + + ociscfg "github.com/owncloud/ocis/v2/ocis-pkg/config" + "github.com/owncloud/ocis/v2/ocis-pkg/shared" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config/defaults" + + "github.com/owncloud/ocis/v2/ocis-pkg/config/envdecode" +) + +// ParseConfig loads configuration from known paths. +func ParseConfig(cfg *config.Config) error { + _, err := ociscfg.BindSourcesToStructs(cfg.Service.Name, cfg) + if err != nil { + return err + } + + defaults.EnsureDefaults(cfg) + + // load all env variables relevant to the config in the current context. + if err := envdecode.Decode(cfg); err != nil { + // no environment variable set for this config is an expected "error" + if !errors.Is(err, envdecode.ErrNoTargetFieldsAreSet) { + return err + } + } + + defaults.Sanitize(cfg) + + return Validate(cfg) +} + +// Validate validates the config +func Validate(cfg *config.Config) error { + if cfg.TokenManager.JWTSecret == "" { + return shared.MissingJWTTokenError(cfg.Service.Name) + } + + return nil +} diff --git a/services/clientlog/pkg/config/service.go b/services/clientlog/pkg/config/service.go new file mode 100644 index 00000000000..d1eac383f0b --- /dev/null +++ b/services/clientlog/pkg/config/service.go @@ -0,0 +1,6 @@ +package config + +// Service defines the available service configuration. +type Service struct { + Name string `yaml:"-"` +} diff --git a/services/clientlog/pkg/config/tracing.go b/services/clientlog/pkg/config/tracing.go new file mode 100644 index 00000000000..66f0332ec15 --- /dev/null +++ b/services/clientlog/pkg/config/tracing.go @@ -0,0 +1,21 @@ +package config + +import "github.com/owncloud/ocis/v2/ocis-pkg/tracing" + +// Tracing defines the available tracing configuration. +type Tracing struct { + Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;USERLOG_TRACING_ENABLED" desc:"Activates tracing."` + Type string `yaml:"type" env:"OCIS_TRACING_TYPE;USERLOG_TRACING_TYPE" desc:"The type of tracing. Defaults to '', which is the same as 'jaeger'. Allowed tracing types are 'jaeger' and '' as of now."` + Endpoint string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;USERLOG_TRACING_ENDPOINT" desc:"The endpoint of the tracing agent."` + Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;USERLOG_TRACING_COLLECTOR" desc:"The HTTP endpoint for sending spans directly to a collector, i.e. http://jaeger-collector:14268/api/traces. Only used if the tracing endpoint is unset."` +} + +// Convert Tracing to the tracing package's Config struct. +func (t Tracing) Convert() tracing.Config { + return tracing.Config{ + Enabled: t.Enabled, + Type: t.Type, + Endpoint: t.Endpoint, + Collector: t.Collector, + } +} diff --git a/services/clientlog/pkg/logging/logging.go b/services/clientlog/pkg/logging/logging.go new file mode 100644 index 00000000000..516b592463b --- /dev/null +++ b/services/clientlog/pkg/logging/logging.go @@ -0,0 +1,17 @@ +package logging + +import ( + "github.com/owncloud/ocis/v2/ocis-pkg/log" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" +) + +// LoggerFromConfig initializes a service-specific logger instance. +func Configure(name string, cfg *config.Log) log.Logger { + return log.NewLogger( + log.Name(name), + log.Level(cfg.Level), + log.Pretty(cfg.Pretty), + log.Color(cfg.Color), + log.File(cfg.File), + ) +} diff --git a/services/clientlog/pkg/metrics/metrics.go b/services/clientlog/pkg/metrics/metrics.go new file mode 100644 index 00000000000..c1f188f2ae2 --- /dev/null +++ b/services/clientlog/pkg/metrics/metrics.go @@ -0,0 +1,35 @@ +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +var ( + // Namespace defines the namespace for the defines metrics. + Namespace = "ocis" + + // Subsystem defines the subsystem for the defines metrics. + Subsystem = "clientlog" +) + +// Metrics defines the available metrics of this service. +type Metrics struct { + BuildInfo *prometheus.GaugeVec +} + +// New initializes the available metrics. +func New() *Metrics { + m := &Metrics{ + BuildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: Subsystem, + Name: "build_info", + Help: "Build information", + }, []string{"version"}), + } + + _ = prometheus.Register( + m.BuildInfo, + ) + + // TODO: implement metrics + return m +} diff --git a/services/clientlog/pkg/service/options.go b/services/clientlog/pkg/service/options.go new file mode 100644 index 00000000000..26509a3ed0e --- /dev/null +++ b/services/clientlog/pkg/service/options.go @@ -0,0 +1,65 @@ +package service + +import ( + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + "github.com/cs3org/reva/v2/pkg/events" + "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" + "github.com/owncloud/ocis/v2/ocis-pkg/log" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" + "go.opentelemetry.io/otel/trace" +) + +// Option for the clientlog service +type Option func(*Options) + +// Options for the clientlog service +type Options struct { + Logger log.Logger + Stream events.Stream + Config *config.Config + GatewaySelector pool.Selectable[gateway.GatewayAPIClient] + RegisteredEvents []events.Unmarshaller + TraceProvider trace.TracerProvider +} + +// Logger configures a logger for the clientlog service +func Logger(log log.Logger) Option { + return func(o *Options) { + o.Logger = log + } +} + +// Stream configures an event stream for the clientlog service +func Stream(s events.Stream) Option { + return func(o *Options) { + o.Stream = s + } +} + +// Config adds the config for the clientlog service +func Config(c *config.Config) Option { + return func(o *Options) { + o.Config = c + } +} + +// GatewaySelector adds a grpc client selector for the gateway service +func GatewaySelector(gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) Option { + return func(o *Options) { + o.GatewaySelector = gatewaySelector + } +} + +// RegisteredEvents registers the events the service should listen to +func RegisteredEvents(e []events.Unmarshaller) Option { + return func(o *Options) { + o.RegisteredEvents = e + } +} + +// TraceProvider adds a tracer provider for the clientlog service +func TraceProvider(tp trace.TracerProvider) Option { + return func(o *Options) { + o.TraceProvider = tp + } +} diff --git a/services/clientlog/pkg/service/service.go b/services/clientlog/pkg/service/service.go new file mode 100644 index 00000000000..4f9f6bdfd3b --- /dev/null +++ b/services/clientlog/pkg/service/service.go @@ -0,0 +1,134 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + "github.com/cs3org/reva/v2/pkg/events" + "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/v2/pkg/utils" + "github.com/owncloud/ocis/v2/ocis-pkg/log" + "github.com/owncloud/ocis/v2/services/clientlog/pkg/config" + "go.opentelemetry.io/otel/trace" +) + +// ClientlogService is the service responsible for user activities +type ClientlogService struct { + log log.Logger + cfg *config.Config + gatewaySelector pool.Selectable[gateway.GatewayAPIClient] + registeredEvents map[string]events.Unmarshaller // ? + tp trace.TracerProvider + tracer trace.Tracer + publisher events.Publisher + ch <-chan events.Event +} + +// NewClientlogService returns a clientlog service +func NewClientlogService(opts ...Option) (*ClientlogService, error) { + o := &Options{} + for _, opt := range opts { + opt(o) + } + + if o.Stream == nil { + return nil, fmt.Errorf("need non nil stream (%v) to work properly", o.Stream) + } + + ch, err := events.Consume(o.Stream, "clientlog", o.RegisteredEvents...) + if err != nil { + return nil, err + } + + cl := &ClientlogService{ + log: o.Logger, + cfg: o.Config, + gatewaySelector: o.GatewaySelector, + registeredEvents: make(map[string]events.Unmarshaller), + tp: o.TraceProvider, + tracer: o.TraceProvider.Tracer("github.com/owncloud/ocis/services/clientlog/pkg/service"), + publisher: o.Stream, + ch: ch, + } + + for _, e := range o.RegisteredEvents { + typ := reflect.TypeOf(e) + cl.registeredEvents[typ.String()] = e + } + + return cl, nil +} + +// Run runs the service +func (cl *ClientlogService) Run() error { + for event := range cl.ch { + cl.processEvent(event) + } + + return nil +} + +func (cl *ClientlogService) processEvent(event events.Event) { + gwc, err := cl.gatewaySelector.Next() + if err != nil { + cl.log.Error().Err(err).Interface("event", event).Msg("error getting gatway client") + return + } + + ctx, err := utils.GetServiceUserContext(cl.cfg.ServiceAccount.ServiceAccountID, gwc, cl.cfg.ServiceAccount.ServiceAccountSecret) + if err != nil { + cl.log.Error().Err(err).Interface("event", event).Msg("error authenticating service user") + return + } + + var users []string + switch e := event.Event.(type) { + default: + err = errors.New("unhandled event") + case events.UploadReady: + info, err := utils.GetResource(ctx, e.FileRef, gwc) + if err != nil { + cl.log.Error().Err(err).Interface("event", event).Msg("error getting resource") + return + } + + users, err = utils.GetSpaceMembers(ctx, info.GetSpace().GetId().GetOpaqueId(), gwc, utils.ViewerRole) + if err != nil { + cl.log.Error().Err(err).Interface("event", event).Msg("error getting space members") + return + } + } + + if err != nil { + cl.log.Info().Err(err).Interface("event", event).Msg("error gathering members for event") + return + } + + // II) instruct sse service to send the information + for _, id := range users { + if err := cl.sendSSE(id, event); err != nil { + cl.log.Error().Err(err).Str("userID", id).Str("eventid", event.ID).Msg("failed to store event for user") + return + } + } +} + +func (cl *ClientlogService) sendSSE(userid string, event events.Event) error { + // TODO: convert event + ev := event + + b, err := json.Marshal(ev) + if err != nil { + return err + } + + return events.Publish(context.Background(), cl.publisher, events.SendSSE{ + UserID: userid, + Type: "clientlog-notification", + Message: b, + }) +} diff --git a/services/clientlog/pkg/service/service_suit_test.go b/services/clientlog/pkg/service/service_suit_test.go new file mode 100644 index 00000000000..ab7fb7b1989 --- /dev/null +++ b/services/clientlog/pkg/service/service_suit_test.go @@ -0,0 +1,26 @@ +package service_test + +import ( + "testing" + + "github.com/owncloud/ocis/v2/ocis-pkg/registry" + mRegistry "go-micro.dev/v4/registry" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func init() { + registry.Configure("memory") + r := registry.GetRegistry() + service := registry.BuildGRPCService("com.owncloud.api.gateway", "", "", "") + service.Nodes = []*mRegistry.Node{{ + Address: "any", + }} + + _ = r.Register(service) +} +func TestSearch(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Userlog service Suite") +} diff --git a/services/clientlog/pkg/service/service_test.go b/services/clientlog/pkg/service/service_test.go new file mode 100644 index 00000000000..922fe8e45c9 --- /dev/null +++ b/services/clientlog/pkg/service/service_test.go @@ -0,0 +1,3 @@ +package service_test + +// TODO: TEST! diff --git a/services/sse/README.md b/services/sse/README.md index ac5e43ae415..332d052dd0b 100644 --- a/services/sse/README.md +++ b/services/sse/README.md @@ -2,6 +2,13 @@ The `sse` service is responsible for sending sse (Server-Sent Events) to a user. See [What is Server-Sent Events](https://medium.com/yemeksepeti-teknoloji/what-is-server-sent-events-sse-and-how-to-implement-it-904938bffd73) for a simple introduction and examples of server sent events. +## The `...log` service ecosystem + +`...log` services (`userlog`, `clientlog`) are responsible for composing notifications for a certain audience. + - `userlog` service translates and adjust messages to be human readable + - `clientlog` service composes machine readable messages so clients can act without needing to query the server + - `sse` service is only responsible for sending these messages. It does not care about their form or language + ## Subscribing Clients can subscribe to the `/sse` endpoint to be informed by the server when an event happens. The `sse` endpoint will respect language changes of the user without needing to reconnect. Note that SSE has a limitation of six open connections per browser which can be reached if one has opened various tabs of the Web UI pointing to the same Infinite Scale instance. diff --git a/services/userlog/README.md b/services/userlog/README.md index aaaf18f4091..e89e1dbc5f7 100644 --- a/services/userlog/README.md +++ b/services/userlog/README.md @@ -2,6 +2,13 @@ The `userlog` service is a mediator between the `eventhistory` service and clients who want to be informed about user related events. It provides an API to retrieve those. +## The `...log` service ecosystem + +`...log` services (`userlog`, `clientlog`) are responsible for composing notifications for a certain audience. + - `userlog` service translates and adjust messages to be human readable + - `clientlog` service composes machine readable messages so clients can act without needing to query the server + - `sse` service is only responsible for sending these messages. It does not care about their form or language + ## Prerequisites Running the `userlog` service without running the `eventhistory` service is not possible.