diff --git a/agent/build/agent.toml b/agent/build/agent.toml new file mode 100644 index 00000000..9271bc6d --- /dev/null +++ b/agent/build/agent.toml @@ -0,0 +1,15 @@ +agent-id = "agent1" + +[log] +level = 5 +type = "stdout" + +[server] +address = "0.0.0.0:9008" +tls-cert-file = "/etc/beegfs/cert.pem" +tls-key-file = "/etc/beegfs/key.pem" + +[reconciler] +deployment-strategy = "default" +manifest-path = "/etc/beegfs/manifest.yaml" +active-manifest-path = "/etc/beegfs/.active.manifest.yaml" diff --git a/agent/cmd/beegfs-agent/main.go b/agent/cmd/beegfs-agent/main.go new file mode 100644 index 00000000..ae037584 --- /dev/null +++ b/agent/cmd/beegfs-agent/main.go @@ -0,0 +1,122 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "os/signal" + "syscall" + + "github.com/spf13/pflag" + "github.com/thinkparq/beegfs-go/agent/internal/config" + "github.com/thinkparq/beegfs-go/agent/internal/server" + "github.com/thinkparq/beegfs-go/agent/pkg/reconciler" + "github.com/thinkparq/beegfs-go/common/configmgr" + "github.com/thinkparq/beegfs-go/common/logger" + "go.uber.org/zap" +) + +const ( + envVarPrefix = "BEEAGENT_" +) + +// Set by the build process using ldflags. +var ( + binaryName = "unknown" + version = "unknown" + commit = "unknown" + buildTime = "unknown" +) + +func main() { + pflag.Bool("version", false, "Print the version then exit.") + pflag.String("cfg-file", "/etc/beegfs/agent.toml", "The path to the a configuration file (can be omitted to set all configuration using flags and/or environment variables). When Remote Storage Targets are configured using a file, they can be updated without restarting the application.") + pflag.String("agent-id", "0", "A unique ID used to identify what services from the manifest this agent is responsible for. Should not change after initially starting the agent.") + pflag.String("log.type", "stderr", "Where log messages should be sent ('stderr', 'stdout', 'syslog', 'logfile').") + pflag.String("log.file", "/var/log/beegfs/beegfs-remote.log", "The path to the desired log file when logType is 'log.file' (if needed the directory and all parent directories will be created).") + pflag.Int8("log.level", 3, "Adjust the logging level (0=Fatal, 1=Error, 2=Warn, 3=Info, 4+5=Debug).") + pflag.Int("log.max-size", 1000, "When log.type is 'logfile' the maximum size of the log.file in megabytes before it is rotated.") + pflag.Int("log.num-rotated-files", 5, "When log.type is 'logfile' the maximum number old log.file(s) to keep when log.max-size is reached and the log is rotated.") + pflag.Bool("log.developer", false, "Enable developer logging including stack traces and setting the equivalent of log.level=5 and log.type=stdout (all other log settings are ignored).") + pflag.String("server.address", "0.0.0.0:9008", "The hostname:port where this Agent should listen for requests from the BeeGFS CTL tool.") + pflag.String("server.tls-cert-file", "/etc/beegfs/cert.pem", "Path to a certificate file that provides the identify of this Agent's gRPC server.") + pflag.String("server.tls-key-file", "/etc/beegfs/key.pem", "Path to the key file belonging to the certificate for this Agent's gRPC server.") + pflag.Bool("server.tls-disable", false, "Disable TLS entirely for gRPC communication to this Agent's gRPC server.") + pflag.String("reconciler.manifest-path", "/etc/beegfs/manifest.yaml", "The path to the BeeGFS manifest this agent should apply. The manifest will be identical to the active manifest if applied successfully.") + pflag.String("reconciler.active-manifest-path", "/etc/beegfs/.active.manifest.yaml", "The past to the last BeeGFS manifest successfully applied by this agent.") + pflag.String("reconciler.deployment-strategy", "default", "The deployment strategy used by the reconciler.") + pflag.Bool("developer.dump-config", false, "Dump the full configuration and immediately exit.") + pflag.CommandLine.MarkHidden("developer.dump-config") + pflag.CommandLine.SortFlags = false + pflag.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) + pflag.PrintDefaults() + helpText := ` +Further info: + Configuration may be set using a mix of flags, environment variables, and values from a TOML configuration file. + Configuration will be merged using the following precedence order (highest->lowest): (1) flags (2) environment variables (3) configuration file (4) defaults. +Using environment variables: + To specify configuration using environment variables specify %sKEY=VALUE where KEY is the flag name you want to specify in all capitals replacing dots (.) with a double underscore (__) and hyphens (-) with an underscore (_). + Examples: + export %sLOG__DEBUG=true +` + fmt.Fprintf(os.Stderr, helpText, envVarPrefix, envVarPrefix) + os.Exit(0) + } + pflag.Parse() + + if printVersion, _ := pflag.CommandLine.GetBool("version"); printVersion { + fmt.Printf("%s %s (commit: %s, built: %s)\n", binaryName, version, commit, buildTime) + os.Exit(0) + } + + cfgMgr, err := configmgr.New(pflag.CommandLine, envVarPrefix, &config.AppConfig{}) + if err != nil { + log.Fatalf("unable to get initial configuration: %s", err) + } + c := cfgMgr.Get() + initialCfg, ok := c.(*config.AppConfig) + if !ok { + log.Fatalf("configuration manager returned invalid configuration (expected Agent application configuration)") + } + if initialCfg.Developer.DumpConfig { + fmt.Printf("Dumping AppConfig and exiting...\n\n") + fmt.Printf("%+v\n", initialCfg) + os.Exit(0) + } + + logger, err := logger.New(initialCfg.Log) + if err != nil { + log.Fatalf("unable to initialize logger: %s", err) + } + defer logger.Sync() + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, syscall.SIGINT) + + reconciler, err := reconciler.New(ctx, initialCfg.AgentID, logger.Logger, initialCfg.Reconciler) + if err != nil { + logger.Fatal("unable to initialize reconciler", zap.Error(err)) + } + cfgMgr.AddListener(reconciler) + agentServer, err := server.New(logger.Logger, initialCfg.Server, reconciler) + if err != nil { + logger.Fatal("unable to initialize gRPC server", zap.Error(err)) + } + + errChan := make(chan error, 2) + agentServer.ListenAndServe(errChan) + go cfgMgr.Manage(ctx, logger.Logger) + + select { + case err := <-errChan: + logger.Error("component terminated unexpectedly", zap.Error(err)) + case <-ctx.Done(): + logger.Info("shutdown signal received") + } + cancel() + agentServer.Stop() + if err := reconciler.Stop(); err != nil { + logger.Error("error stopping reconciler", zap.Error(err)) + } + logger.Info("shutdown all components, exiting") +} diff --git a/agent/internal/config/config.go b/agent/internal/config/config.go new file mode 100644 index 00000000..c912b513 --- /dev/null +++ b/agent/internal/config/config.go @@ -0,0 +1,35 @@ +package config + +import ( + "github.com/thinkparq/beegfs-go/agent/internal/server" + "github.com/thinkparq/beegfs-go/agent/pkg/reconciler" + "github.com/thinkparq/beegfs-go/common/configmgr" + "github.com/thinkparq/beegfs-go/common/logger" +) + +type AppConfig struct { + AgentID string `mapstructure:"agent-id"` + Log logger.Config `mapstructure:"log"` + Reconciler reconciler.Config `mapstructure:"reconciler"` + Server server.Config `mapstructure:"server"` + Developer struct { + DumpConfig bool `mapstructure:"dump-config"` + } +} + +func (c *AppConfig) NewEmptyInstance() configmgr.Configurable { + return new(AppConfig) +} + +func (c *AppConfig) UpdateAllowed(newConfig configmgr.Configurable) error { + return nil +} + +func (c *AppConfig) ValidateConfig() error { + return nil +} + +// GetReconcilerConfig returns only the part of an AppConfig expected by the reconciler. +func (c *AppConfig) GetReconcilerConfig() reconciler.Config { + return c.Reconciler +} diff --git a/agent/internal/server/server.go b/agent/internal/server/server.go new file mode 100644 index 00000000..e1ca73c7 --- /dev/null +++ b/agent/internal/server/server.go @@ -0,0 +1,143 @@ +package server + +import ( + "context" + "errors" + "fmt" + "net" + "path" + "reflect" + "sync" + + "github.com/thinkparq/beegfs-go/agent/pkg/manifest" + "github.com/thinkparq/beegfs-go/agent/pkg/reconciler" + pb "github.com/thinkparq/protobuf/go/agent" + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" +) + +type Config struct { + Address string `mapstructure:"address"` + TlsCertFile string `mapstructure:"tls-cert-file"` + TlsKeyFile string `mapstructure:"tls-key-file"` + TlsDisable bool `mapstructure:"tls-disable"` +} + +type AgentServer struct { + pb.UnimplementedBeeAgentServer + log *zap.Logger + wg *sync.WaitGroup + Config + grpcServer *grpc.Server + reconciler reconciler.Reconciler +} + +func New(log *zap.Logger, config Config, reconciler reconciler.Reconciler) (*AgentServer, error) { + log = log.With(zap.String("component", path.Base(reflect.TypeOf(AgentServer{}).PkgPath()))) + + s := AgentServer{ + log: log, + Config: config, + wg: new(sync.WaitGroup), + reconciler: reconciler, + } + var grpcServerOpts []grpc.ServerOption + if !s.TlsDisable && s.TlsCertFile != "" && s.TlsKeyFile != "" { + creds, err := credentials.NewServerTLSFromFile(s.TlsCertFile, s.TlsKeyFile) + if err != nil { + return nil, err + } + grpcServerOpts = append(grpcServerOpts, grpc.Creds(creds)) + } else { + s.log.Warn("not using TLS because it was explicitly disabled or a certificate and/or key were not specified") + } + s.grpcServer = grpc.NewServer(grpcServerOpts...) + pb.RegisterBeeAgentServer(s.grpcServer, &s) + return &s, nil +} + +func (s *AgentServer) ListenAndServe(errChan chan<- error) { + go func() { + s.log.Info("listening on local network address", zap.Any("address", s.Address)) + lis, err := net.Listen("tcp", s.Address) + if err != nil { + errChan <- fmt.Errorf("remote server: error listening on the specified address %s: %w", s.Address, err) + return + } + s.log.Info("serving gRPC requests") + err = s.grpcServer.Serve(lis) + if err != nil { + errChan <- fmt.Errorf("remote server: error serving gRPC requests: %w", err) + } + }() +} + +func (s *AgentServer) Stop() { + s.log.Info("attempting to stop gRPC server") + s.grpcServer.Stop() + s.wg.Wait() +} + +func (s *AgentServer) UpdateManifest(ctx context.Context, request *pb.UpdateManifestRequest) (*pb.UpdateManifestResponse, error) { + s.wg.Add(1) + defer s.wg.Done() + + filesystems := make(map[string]manifest.Filesystem, len(request.GetConfig())) + for fsUUID, protoFS := range request.GetConfig() { + if protoFS == nil { + return nil, status.Error(codes.InvalidArgument, "file system configuration was unexpectedly nil for fsUUID "+fsUUID) + } + filesystems[fsUUID] = manifest.FromProto(protoFS) + } + + if err := s.reconciler.UpdateConfiguration(manifest.Manifest{ + Filesystems: filesystems, + }); err != nil { + return nil, grpcStatusFrom(err) + } + return &pb.UpdateManifestResponse{ + AgentId: s.reconciler.GetAgentID(), + }, nil +} + +func (s *AgentServer) ReconciliationStatus(ctx context.Context, request *pb.ReconciliationStatusRequest) (*pb.ReconciliationStatusResponse, error) { + s.wg.Add(1) + defer s.wg.Done() + if result, err := s.reconciler.Status(); err != nil { + return nil, grpcStatusFrom(err) + } else { + return &pb.ReconciliationStatusResponse{ + Status: result.Status, + AgentId: s.reconciler.GetAgentID(), + }, nil + } +} + +func (s *AgentServer) CancelReconciliation(ctx context.Context, request *pb.CancelReconciliationRequest) (*pb.CancelReconciliationResponse, error) { + s.wg.Add(1) + defer s.wg.Done() + if result, err := s.reconciler.Cancel(request.GetReason()); err != nil { + return nil, grpcStatusFrom(err) + } else { + return &pb.CancelReconciliationResponse{ + Status: result.Status, + AgentId: s.reconciler.GetAgentID(), + }, nil + } +} + +func grpcStatusFrom(err error) error { + var grpcErr error + switch { + case errors.Is(err, reconciler.ErrSavingManifest): + grpcErr = status.Error(codes.FailedPrecondition, err.Error()) + case errors.Is(err, reconciler.ErrBadManifest): + grpcErr = status.Error(codes.InvalidArgument, err.Error()) + default: + grpcErr = status.Error(codes.Unknown, err.Error()) + } + return grpcErr +} diff --git a/agent/pkg/deploy/deploy.go b/agent/pkg/deploy/deploy.go new file mode 100644 index 00000000..3cebf10c --- /dev/null +++ b/agent/pkg/deploy/deploy.go @@ -0,0 +1,44 @@ +package deploy + +import "context" + +// Deployer is responsible for carrying out the steps needed to manage a BeeGFS "service" and +// handles starting/modifying/stopping various system resources. +type Deployer interface { + Installer + Networker + Mounter + Servicer + // Cleanup should be called once the deployer is no longer needed to cleanup any long lived + // resources created by setting up a particular deployment strategy. + Cleanup() error +} + +func NewDefaultStrategy(ctx context.Context) (Deployer, error) { + packageManager, err := DetectPackageManager() + if err != nil { + return nil, err + } + + systemd, err := NewSystemd(ctx) + if err != nil { + return nil, err + } + return &defaultStrategy{ + Package: packageManager, + IP: IP{}, + Mount: Mount{}, + Systemd: systemd, + }, nil +} + +type defaultStrategy struct { + Package // implements Sourcerer + IP // implements Networker + Mount // implements Mounter + Systemd // implements Servicer +} + +func (s *defaultStrategy) Cleanup() error { + return s.Systemd.Cleanup() +} diff --git a/agent/pkg/deploy/install.go b/agent/pkg/deploy/install.go new file mode 100644 index 00000000..31af6e53 --- /dev/null +++ b/agent/pkg/deploy/install.go @@ -0,0 +1,89 @@ +package deploy + +import ( + "context" + "errors" + "fmt" + "os/exec" + + "github.com/thinkparq/beegfs-go/agent/pkg/manifest" +) + +type Installer interface { + ApplySourceRepo(ctx context.Context, add manifest.InstallSource) error + DeleteSourceRepo(ctx context.Context, remove manifest.InstallSource) error + ApplyInstall(ctx context.Context, ref string) error + DeleteInstall(ctx context.Context, ref string) error +} + +func DetectPackageManager() (Package, error) { + isExecutableInPath := func(name string) bool { + _, err := exec.LookPath(name) + return err == nil + } + switch { + case isExecutableInPath("apt"): + return Package{ + manager: &AptPackage{}, + }, nil + } + return Package{}, fmt.Errorf("detecting package manager: unsupported or undetectable package manager") +} + +// Package provides the ability to install BeeGFS using the package manager. It implements any +// general functionality and defers to the actual manager based on the specific distribution. +type Package struct { + manager Installer + // isLocal is set if the manifest specifies the source type is local. This indicates all package + // manager operations should be a no-op for this FS in the manifest. This allows the manifest to + // fully control the installation source independent of the deployment strategy for each agent. + isLocal bool +} + +func (p *Package) ApplySourceRepo(ctx context.Context, add manifest.InstallSource) error { + if add.Type == manifest.LocalInstall { + p.isLocal = true + return nil + } + return p.manager.ApplySourceRepo(ctx, add) +} + +func (p *Package) DeleteSourceRepo(ctx context.Context, remove manifest.InstallSource) error { + if remove.Type == manifest.LocalInstall { + p.isLocal = false + return nil + } + return p.manager.DeleteSourceRepo(ctx, remove) +} + +func (p *Package) ApplyInstall(ctx context.Context, ref string) error { + if p.isLocal { + return nil + } + return p.manager.ApplyInstall(ctx, ref) +} + +func (p *Package) DeleteInstall(ctx context.Context, ref string) error { + if p.isLocal { + return nil + } + return p.manager.DeleteInstall(ctx, ref) +} + +type AptPackage struct{} + +func (p *AptPackage) ApplySourceRepo(ctx context.Context, add manifest.InstallSource) error { + return errors.New("not implemented") +} + +func (p *AptPackage) DeleteSourceRepo(ctx context.Context, remove manifest.InstallSource) error { + return errors.New("not implemented") +} + +func (p *AptPackage) ApplyInstall(ctx context.Context, ref string) error { + return errors.New("not implemented") +} + +func (p *AptPackage) DeleteInstall(ctx context.Context, ref string) error { + return errors.New("not implemented") +} diff --git a/agent/pkg/deploy/mount.go b/agent/pkg/deploy/mount.go new file mode 100644 index 00000000..171f51a1 --- /dev/null +++ b/agent/pkg/deploy/mount.go @@ -0,0 +1,43 @@ +package deploy + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + + "github.com/thinkparq/beegfs-go/agent/pkg/manifest" +) + +type Mounter interface { + ApplyTargets(ctx context.Context, add []manifest.Target) error + DestroyTargets(ctx context.Context, remove []manifest.Target) error +} + +type Mount struct { +} + +func (m *Mount) ApplyTargets(ctx context.Context, add []manifest.Target) error { + for _, target := range add { + if target.ULFS != nil { + return fmt.Errorf("unable to apply target %d: formatting and/or mounting an underlying file system is not implemented yet", target.ID) + } + if err := os.MkdirAll(target.GetPath(), 0700); err != nil { + return fmt.Errorf("unable to apply target %d: unable to create root directory %s: %w", target.ID, target.Path, err) + } + name, args := target.GetInitCmd() + output, err := exec.CommandContext(ctx, name, args...).CombinedOutput() + if err != nil { + if !strings.Contains(string(output), "already exists") { + return fmt.Errorf("unable to initialize target %d: %s (%w)", target.ID, output, err) + } + } + } + return nil +} + +func (m *Mount) DestroyTargets(ctx context.Context, remove []manifest.Target) error { + return errors.New("not implemented") +} diff --git a/agent/pkg/deploy/network.go b/agent/pkg/deploy/network.go new file mode 100644 index 00000000..9a1fc677 --- /dev/null +++ b/agent/pkg/deploy/network.go @@ -0,0 +1,39 @@ +package deploy + +import ( + "context" + "errors" + "fmt" + "os/exec" + "strings" + + "github.com/thinkparq/beegfs-go/agent/pkg/manifest" +) + +type Networker interface { + ApplyInterfaces(ctx context.Context, add []manifest.Nic) error + DestroyInterfaces(ctx context.Context, remove []manifest.Nic) error +} + +type IP struct { +} + +func (i *IP) ApplyInterfaces(ctx context.Context, add []manifest.Nic) error { + for _, nic := range add { + if nic.Addr == "" { + continue // no-op + } + output, err := exec.CommandContext(ctx, "ip", "addr", "show", "dev", nic.Name).Output() + if err != nil { + return fmt.Errorf("unable to verify IP %s is configured for interface %s: %w", nic.Addr, nic.Name, err) + } + if !strings.Contains(string(output), nic.Addr) { + return fmt.Errorf("unable to apply IP %s to interface %s: configuring IPs is not supported yet", nic.Addr, nic.Name) + } + } + return nil +} + +func (i *IP) DestroyInterfaces(ctx context.Context, remove []manifest.Nic) error { + return errors.New("not implemented") +} diff --git a/agent/pkg/deploy/service.go b/agent/pkg/deploy/service.go new file mode 100644 index 00000000..d41ffb1c --- /dev/null +++ b/agent/pkg/deploy/service.go @@ -0,0 +1,83 @@ +package deploy + +import ( + "context" + "fmt" + + "github.com/coreos/go-systemd/v22/dbus" + "github.com/thinkparq/beegfs-go/agent/pkg/manifest" +) + +type Servicer interface { + ApplyService(ctx context.Context, add manifest.Service) error + DestroyService(ctx context.Context, remove manifest.Service) error +} + +func NewSystemd(ctx context.Context) (Systemd, error) { + conn, err := dbus.NewSystemConnectionContext(ctx) + if err != nil { + return Systemd{}, fmt.Errorf("unable to connect to the system bus: %w", err) + } + return Systemd{ + conn: conn, + }, nil + +} + +// Systemd provides a method to deploy BeeGFS services using systemd. +type Systemd struct { + conn *dbus.Conn +} + +func (d *Systemd) Cleanup() error { + d.conn.Close() + return nil +} + +func (d *Systemd) ApplyService(ctx context.Context, add manifest.Service) error { + + if err := d.DestroyService(ctx, add); err != nil { + return fmt.Errorf("error destroying existing service to apply updates: %w", err) + } + + cmd := append([]string{add.Executable}, add.GetConfig()...) + properties := []dbus.Property{ + dbus.PropExecStart(cmd, false), + dbus.PropDescription(add.GetDescription()), + dbus.PropRemainAfterExit(false), + dbus.PropType("simple"), + } + _, err := d.conn.StartTransientUnitContext(ctx, add.GetSystemdUnit(), "replace", properties, nil) + if err != nil { + return fmt.Errorf("failed to start transient unit %s: %w", add.GetSystemdUnit(), err) + } + return nil +} + +func (d *Systemd) DestroyService(ctx context.Context, remove manifest.Service) error { + units, err := d.conn.ListUnitsByNamesContext(ctx, []string{remove.GetSystemdUnit()}) + if err != nil { + return fmt.Errorf("error querying systemd units: %w", err) + } + + if len(units) == 0 || units[0].LoadState == "not-found" { + return nil + } + + ch := make(chan string) + _, err = d.conn.StopUnitContext(ctx, remove.GetSystemdUnit(), "replace", ch) + if err != nil { + return err + } + select { + case <-ch: + if units[0].SubState == "failed" { + if err := d.conn.ResetFailedUnitContext(ctx, remove.GetSystemdUnit()); err != nil { + return fmt.Errorf("error resetting failed unit context: %w", err) + } + } + return nil + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/agent/pkg/manifest/common.go b/agent/pkg/manifest/common.go new file mode 100644 index 00000000..7125d430 --- /dev/null +++ b/agent/pkg/manifest/common.go @@ -0,0 +1,205 @@ +package manifest + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/thinkparq/beegfs-go/common/beegfs" + pb "github.com/thinkparq/protobuf/go/agent" +) + +const ( + DefaultExecutablePath = "/opt/beegfs/sbin/" +) + +type Common struct { + Auth *Auth `yaml:"auth"` + TLS *TLS `yaml:"tls"` + GlobalConfig ServiceConfigs `yaml:"config"` + InstallSource InstallSource `yaml:"install-source"` +} + +type Auth struct { + Secret string `yaml:"secret"` +} + +type TLS struct { + Key string `yaml:"key"` + Cert string `yaml:"cert"` +} + +type ServiceConfigs map[beegfs.NodeType]map[string]string + +func (s *ServiceConfigs) UnmarshalYAML(unmarshal func(any) error) error { + // We cannot directly apply validation to map[beegfs.NodeType]... during unmarshal because the + // YAML input uses string keys and as a result things blow up (spectacularly). + intermediate := map[string]map[string]string{} + if err := unmarshal(&intermediate); err != nil { + return err + } + + result := make(ServiceConfigs, len(intermediate)) + for key, val := range intermediate { + nodeType := beegfs.NodeTypeFromString(key) + if nodeType == beegfs.InvalidNodeType { + return fmt.Errorf("invalid node type '%s' in config", key) + } + result[nodeType] = val + } + + *s = result + return nil +} + +func (c ServiceConfigs) toProto() []*pb.ServiceConfig { + pbServiceConfigs := make([]*pb.ServiceConfig, 0, len(c)) + for nodeType, serviceMap := range c { + pbServiceConfigs = append(pbServiceConfigs, &pb.ServiceConfig{ + ServiceType: *nodeType.ToProto(), + StringMap: serviceMap, + }) + } + return pbServiceConfigs +} + +func serviceConfigsFromProto(m []*pb.ServiceConfig) ServiceConfigs { + nsm := make(ServiceConfigs, len(m)) + for _, service := range m { + if service != nil && service.GetStringMap() != nil { + nsm[beegfs.NodeTypeFromProto(service.ServiceType)] = service.GetStringMap() + } + } + return nsm +} + +type InstallSource struct { + Type InstallType `yaml:"type"` + Repo string `yaml:"repo"` + Refs SourceRefs `yaml:"refs"` +} + +// nodeTypeToExecutablePath takes a node type and returns the default path to that node's binary. +func (s InstallSource) nodeTypeToExecutablePath(nodeType beegfs.NodeType) string { + switch nodeType { + case beegfs.Management: + return filepath.Clean(DefaultExecutablePath + "beegfs-mgmtd") + default: + return filepath.Clean(DefaultExecutablePath + nodeType.String()) + } +} + +// refToExecutablePath takes a reference to a package, container image, etc. and generates a default +// executable path based on the install source type and reference string format. +func (s InstallSource) refToExecutablePath(ref string) string { + switch s.Type { + case PackageInstall, LocalInstall: + if r := strings.Split(ref, "="); len(r) == 2 { + return filepath.Clean(DefaultExecutablePath + r[0]) + } + return filepath.Clean(DefaultExecutablePath + ref) + default: + return ref + } +} + +type SourceRefs map[beegfs.NodeType]string + +func (s *SourceRefs) UnmarshalYAML(unmarshal func(any) error) error { + // We cannot directly apply validation to map[beegfs.NodeType]... during unmarshal because the + // YAML input uses string keys and as a result things blow up (spectacularly). + intermediate := map[string]string{} + if err := unmarshal(&intermediate); err != nil { + return err + } + + result := make(SourceRefs, len(intermediate)) + for key, val := range intermediate { + nodeType := beegfs.NodeTypeFromString(key) + if nodeType == beegfs.InvalidNodeType { + return fmt.Errorf("invalid node type '%s' in source refs", key) + } + result[nodeType] = val + } + + *s = result + return nil +} + +func (c SourceRefs) toProto() []*pb.SourceRef { + pbSourceRefs := make([]*pb.SourceRef, 0, len(c)) + for nodeType, ref := range c { + pbSourceRefs = append(pbSourceRefs, &pb.SourceRef{ + ServiceType: *nodeType.ToProto(), + Ref: ref, + }) + } + return pbSourceRefs +} + +func sourceRefsFromProto(r []*pb.SourceRef) SourceRefs { + srs := make(SourceRefs, len(r)) + for _, ref := range r { + if ref != nil { + srs[beegfs.NodeTypeFromProto(ref.ServiceType)] = ref.GetRef() + } + } + return srs +} + +type InstallType int + +const ( + UnknownInstall InstallType = iota + LocalInstall + PackageInstall +) + +func (s InstallType) ToProto() pb.InstallType { + switch s { + case LocalInstall: + return pb.InstallType_LOCAL + case PackageInstall: + return pb.InstallType_PACKAGE + default: + return pb.InstallType_UNKNOWN + } +} + +func sourceTypeFromProto(st pb.InstallType) InstallType { + switch st { + case pb.InstallType_LOCAL: + return LocalInstall + case pb.InstallType_PACKAGE: + return PackageInstall + default: + return UnknownInstall + } +} + +func (s *InstallType) UnmarshalYAML(unmarshal func(any) error) error { + var str string + if err := unmarshal(&str); err != nil { + return err + } + switch str { + case "local": + *s = LocalInstall + case "package": + *s = PackageInstall + default: + *s = UnknownInstall + } + return nil +} + +func (s InstallType) MarshalYAML() (any, error) { + switch s { + case LocalInstall: + return "local", nil + case PackageInstall: + return "package", nil + default: + return "unknown", nil + } +} diff --git a/agent/pkg/manifest/filesystem.go b/agent/pkg/manifest/filesystem.go new file mode 100644 index 00000000..d5b9dbd2 --- /dev/null +++ b/agent/pkg/manifest/filesystem.go @@ -0,0 +1,283 @@ +// Package manifest defines Go-native structs for defining a BeeGFS instance. This includes +// functions for converting to/from protobuf messages and loading/unloading from YAML files. +// Protobuf structs are not used directly (as is done in other BeeGFS Go projects) to provide a more +// user-friendly YAML manifest than what protobuf generated structs allow. +package manifest + +import ( + "fmt" + "os" + + "github.com/google/uuid" + "github.com/thinkparq/beegfs-go/common/beegfs" + pb "github.com/thinkparq/protobuf/go/agent" + "gopkg.in/yaml.v3" +) + +const ShortUUIDLen = 8 + +func ShortUUID(u uuid.UUID) string { + return u.String()[:ShortUUIDLen] +} + +type Filesystem struct { + Agents map[string]Agent `yaml:"agents"` + Common Common `yaml:"common"` +} + +type Agent struct { + Services []Service `yaml:"services"` + // Global agent interfaces potentially reused by multiple services. + Interfaces []Nic `yaml:"interfaces"` +} + +type Nic struct { + Name string `yaml:"name"` + Addr string `yaml:"address"` +} + +// InheritGlobalConfig accepts a shortUUID used internally to generate globally unique names and +// identifiers in case resources for multiple file systems exist on the same machine. Derived by +// taking the first ShortUUIDLen hex digits of the full 128-bit v4 UUID. The caller is responsible +// for validating the shortUUID including verifying no collisions are possible in this manifest. +func (f *Filesystem) InheritGlobalConfig(shortUUID string, longUUID string) error { + for agentID, agent := range f.Agents { + for i := range agent.Services { + service := &agent.Services[i] + service.shortUUID = shortUUID + service.longUUID = longUUID + // Inherit global interface configuration if there are no service specific interfaces. + if len(service.Interfaces) == 0 { + service.Interfaces = agent.Interfaces + } + // Inherit global service configuration based on the service type. + if commonServiceConfig, ok := f.Common.GlobalConfig[agent.Services[i].Type]; ok { + service.Config = inheritMapDefaults(commonServiceConfig, service.Config) + } + // Inherit global source configuration based on the service type. + if service.Executable == "" { + if ref, ok := f.Common.InstallSource.Refs[service.Type]; ok { + // If there is a global install reference for this service type use this to + // derive the executable path. + service.Executable = f.Common.InstallSource.refToExecutablePath(ref) + } else { + // Otherwise get the default executable path for this service type. + service.Executable = f.Common.InstallSource.nodeTypeToExecutablePath(service.Type) + } + } + // Inherit target configuration from the FS and service: + for t := range service.Targets { + agent.Services[i].Targets[t].longUUID = longUUID + agent.Services[i].Targets[t].shortUUID = shortUUID + agent.Services[i].Targets[t].nodeType = service.Type + // TODO: May be different for each service type. + agent.Services[i].Targets[t].initCmd = service.Executable + } + + if targetConfig, err := service.GetTargetsConfig(); err != nil { + return err + } else { + for k, v := range targetConfig { + if user, ok := service.Config[k]; ok { + return fmt.Errorf("auto-generated target config for %s=%s would overwrite user config %s (user config must be removed)", k, v, user) + } + service.Config[k] = v + } + } + + // TODO: Inherit global conn.auth and TLS config based on service type. Return an error + // if users try to manually specify this in the global or per-service config somehow. + // Maybe this is implemented as methods on the Auth and TLS structs? + } + f.Agents[agentID] = agent + } + return nil +} + +func inheritMapDefaults(defaults, target map[string]string) map[string]string { + if target == nil { + target = make(map[string]string, 0) + } + for k, v := range defaults { + if _, ok := target[k]; !ok { + target[k] = v + } + } + return target +} + +func FromProto(protoFS *pb.Filesystem) Filesystem { + var fs Filesystem + if protoFS == nil { + return fs + } + + pSrc := protoFS.GetCommon().GetInstallSource() + fs.Common = Common{ + GlobalConfig: serviceConfigsFromProto(protoFS.Common.GetGlobalConfig()), + InstallSource: InstallSource{ + Type: sourceTypeFromProto(pSrc.Type), + Repo: pSrc.Repo, + Refs: sourceRefsFromProto(pSrc.Refs), + }, + } + + if protoFS.GetCommon().GetAuth() != nil { + fs.Common.Auth = &Auth{ + Secret: protoFS.GetCommon().GetAuth().GetSecret(), + } + } + + if protoFS.GetCommon().GetTls() != nil { + fs.Common.TLS = &TLS{ + Key: protoFS.GetCommon().GetTls().GetKey(), + Cert: protoFS.GetCommon().GetTls().GetCert(), + } + } + + fs.Agents = make(map[string]Agent, len(protoFS.GetAgent())) + for id, a := range protoFS.GetAgent() { + agent := Agent{ + Services: make([]Service, 0), + Interfaces: make([]Nic, 0), + } + for _, i := range a.GetInterfaces() { + agent.Interfaces = append(agent.Interfaces, Nic{ + Name: i.Name, + Addr: i.Addr, + }) + } + for _, s := range a.GetServices() { + service := Service{ + ID: beegfs.NumId(s.GetNumId()), + Type: beegfs.NodeTypeFromProto(s.ServiceType), + Config: s.GetConfig(), + Interfaces: make([]Nic, 0), + Targets: make([]Target, 0), + Executable: s.GetExecutable(), + } + + for _, i := range s.GetInterfaces() { + service.Interfaces = append(service.Interfaces, Nic{ + Name: i.Name, + Addr: i.Addr, + }) + } + + for _, t := range s.GetTargets() { + target := Target{ + ID: beegfs.NumId(t.GetNumId()), + Path: t.GetPath(), + } + if t.GetUlfs() != nil { + target.ULFS = &UnderlyingFS{ + Device: t.GetUlfs().GetDevice(), + Type: ulfsTypeFromProto(t.GetUlfs().GetType()), + FormatFlags: t.GetUlfs().GetFormatFlags(), + MountFlags: t.GetUlfs().GetMountFlags(), + } + + } + service.Targets = append(service.Targets, target) + } + agent.Services = append(agent.Services, service) + } + fs.Agents[id] = agent + } + return fs +} + +func ToProto(fs *Filesystem) *pb.Filesystem { + pbFS := &pb.Filesystem{ + Common: &pb.Filesystem_Common{ + GlobalConfig: fs.Common.GlobalConfig.toProto(), + InstallSource: &pb.InstallSource{ + Type: fs.Common.InstallSource.Type.ToProto(), + Repo: fs.Common.InstallSource.Repo, + Refs: fs.Common.InstallSource.Refs.toProto(), + }, + }, + Agent: make(map[string]*pb.Agent), + } + + if fs.Common.Auth != nil { + pbFS.Common.Auth = &pb.Auth{ + Secret: fs.Common.Auth.Secret, + } + } + + if fs.Common.TLS != nil { + pbFS.Common.Tls = &pb.TLS{ + Key: fs.Common.TLS.Key, + Cert: fs.Common.TLS.Cert, + } + } + + for agentID, agent := range fs.Agents { + pbAgent := &pb.Agent{ + Services: make([]*pb.Service, 0, len(agent.Services)), + Interfaces: make([]*pb.Nic, 0, len(agent.Interfaces)), + } + for _, i := range agent.Interfaces { + pbAgent.Interfaces = append(pbAgent.Interfaces, &pb.Nic{ + Name: i.Name, + Addr: i.Addr, + }) + } + for _, service := range agent.Services { + pbService := &pb.Service{ + NumId: uint32(service.ID), + ServiceType: *service.Type.ToProto(), + Config: service.Config, + Interfaces: make([]*pb.Nic, 0, len(service.Interfaces)), + Targets: make([]*pb.Target, 0, len(service.Targets)), + Executable: service.Executable, + } + + for _, nic := range service.Interfaces { + pbService.Interfaces = append(pbService.Interfaces, &pb.Nic{ + Name: nic.Name, + Addr: nic.Addr, + }) + } + for _, tgt := range service.Targets { + pbTarget := &pb.Target{ + NumId: uint32(tgt.ID), + Path: tgt.Path, + } + if tgt.ULFS != nil { + pbTarget.Ulfs = &pb.Target_UnderlyingFSOpts{ + Device: tgt.ULFS.Device, + Type: tgt.ULFS.Type.toProto(), + FormatFlags: tgt.ULFS.FormatFlags, + MountFlags: tgt.ULFS.MountFlags, + } + } + pbService.Targets = append(pbService.Targets, pbTarget) + } + pbAgent.Services = append(pbAgent.Services, pbService) + } + pbFS.Agent[agentID] = pbAgent + } + return pbFS +} + +func FromDisk(path string) (Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return Manifest{}, err + } + var manifest Manifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return Manifest{}, err + } + return manifest, nil +} + +func ToDisk(manifest Manifest, path string) error { + data, err := yaml.Marshal(&manifest) + if err != nil { + return err + } + return os.WriteFile(path, data, 0644) +} diff --git a/agent/pkg/manifest/filesystem_test.go b/agent/pkg/manifest/filesystem_test.go new file mode 100644 index 00000000..bf997690 --- /dev/null +++ b/agent/pkg/manifest/filesystem_test.go @@ -0,0 +1,234 @@ +package manifest + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/thinkparq/beegfs-go/common/beegfs" + pb "github.com/thinkparq/protobuf/go/agent" + pbb "github.com/thinkparq/protobuf/go/beegfs" +) + +func TestFromToProto_RoundTrip(t *testing.T) { + original := &pb.Filesystem{ + Common: &pb.Filesystem_Common{ + Auth: &pb.Auth{ + Secret: "secret", + }, + Tls: &pb.TLS{ + Key: "tlsKey", + Cert: "tlsCert", + }, + GlobalConfig: []*pb.ServiceConfig{ + { + ServiceType: pbb.NodeType_META, + StringMap: map[string]string{"key": "val"}, + }, + }, + InstallSource: &pb.InstallSource{ + Type: pb.InstallType_PACKAGE, + Refs: []*pb.SourceRef{ + { + ServiceType: pbb.NodeType_META, + Ref: "ref", + }, + }, + }, + }, + + Agent: map[string]*pb.Agent{ + "agent1": { + Interfaces: []*pb.Nic{ + {Name: "eth0", Addr: "11.0.0.1/16"}, + }, + Services: []*pb.Service{ + { + NumId: 1, + ServiceType: pbb.NodeType_META, + Config: map[string]string{"nkey": "nval"}, + Interfaces: []*pb.Nic{ + {Name: "ib0", Addr: "10.0.0.1/16"}, + }, + Executable: "/opt/beegfs/beegfs-meta", + Targets: []*pb.Target{ + { + NumId: 101, + Path: "/mnt", + Ulfs: &pb.Target_UnderlyingFSOpts{ + Device: "/dev/sda1", + Type: pb.Target_UnderlyingFSOpts_EXT4, + FormatFlags: "force", + MountFlags: "ro", + }, + }, + }, + }, + }, + }, + }, + } + + goStruct := FromProto(original) + roundTripped := ToProto(&goStruct) + + assert.Equal(t, original, roundTripped, "round-trip protobuf -> go -> protobuf did not match original") +} + +func TestInheritGlobalConfig(t *testing.T) { + tests := []struct { + name string + input Filesystem + expectedNIC string // Expected NIC name in service if inherited + expectedCfg map[string]string + expectedExec string + }{ + { + name: "inherit source, NIC and meta config", + input: Filesystem{ + Common: Common{ + GlobalConfig: ServiceConfigs{beegfs.Meta: map[string]string{ + "foo": "bar", // inherited + "baz": "service-specific", // overridden + }}, + InstallSource: InstallSource{ + Refs: SourceRefs{beegfs.Meta: "beegfs-meta=8.0.1"}, + Type: PackageInstall, + Repo: "repoURL", + }, + }, + Agents: map[string]Agent{ + "agent1": { + Interfaces: []Nic{ + {Name: "ib0", Addr: "10.0.0.1/16"}, + }, + Services: []Service{ + { + Type: beegfs.Meta, + ID: 1, + Config: map[string]string{"baz": "service-specific"}, + Targets: []Target{ + { + ID: beegfs.NumId(1), + Path: "/beegfs/", + }, + }, + }, + }, + }, + }, + }, + expectedNIC: "ib0", + expectedCfg: map[string]string{ + "foo": "bar", // inherited + "baz": "service-specific", // overridden + }, + expectedExec: "/opt/beegfs/beegfs-meta", + }, + { + name: "no inheritance if NICs or source are present", + input: Filesystem{ + Common: Common{ + GlobalConfig: ServiceConfigs{ + beegfs.Meta: map[string]string{ + "quota": "enabled", + }, + }, + InstallSource: InstallSource{ + Type: PackageInstall, + Refs: SourceRefs{beegfs.Meta: "beegfs-meta=8.0.1"}, + Repo: "repoURL", + }, + }, + Agents: map[string]Agent{ + "agent1": { + Interfaces: []Nic{ + {Name: "ib0", Addr: "10.0.0.1/16"}, + }, + Services: []Service{ + { + Type: beegfs.Meta, + ID: 2, + Interfaces: []Nic{ + {Name: "eth0", Addr: "192.168.0.1/24"}, + }, + Config: map[string]string{"quota": "override"}, + Executable: "/tmp/beegfs-meta", + }, + }, + }, + }, + }, + expectedNIC: "eth0", + expectedCfg: map[string]string{ + "quota": "override", + }, + expectedExec: "/tmp/beegfs-meta", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fs := tt.input + // 3b6f972b-64c7-4378-9f8e-172cf88c7d93 + fs.InheritGlobalConfig("3b6f972b", "3b6f972b-64c7-4378-9f8e-172cf88c7d93") + agent := fs.Agents["agent1"] + service := agent.Services[0] + assert.Equal(t, tt.expectedNIC, service.Interfaces[0].Name) + assert.Equal(t, tt.expectedCfg, service.Config) + assert.Equal(t, "3b6f972b", service.shortUUID) + for _, target := range service.Targets { + assert.Equal(t, "/beegfs/3b6f972b/meta_1", target.GetPath(), "generated target path did not match") + } + + }) + } +} + +func TestInheritMapDefaults(t *testing.T) { + tests := []struct { + name string + defaults map[string]string + target map[string]string + expected map[string]string + }{ + { + name: "adds missing keys", + defaults: map[string]string{ + "a": "1", + "b": "2", + }, + target: map[string]string{ + "a": "1-overridden", + }, + expected: map[string]string{ + "a": "1-overridden", // should NOT be overridden + "b": "2", // should be added + }, + }, + { + name: "target already has all keys", + defaults: map[string]string{"a": "1"}, + target: map[string]string{"a": "custom"}, + expected: map[string]string{"a": "custom"}, + }, + { + name: "empty defaults", + defaults: map[string]string{}, + target: map[string]string{"a": "existing"}, + expected: map[string]string{"a": "existing"}, + }, + { + name: "empty target", + defaults: map[string]string{"a": "1"}, + target: map[string]string{}, + expected: map[string]string{"a": "1"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := inheritMapDefaults(tt.defaults, tt.target) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/agent/pkg/manifest/manifest.go b/agent/pkg/manifest/manifest.go new file mode 100644 index 00000000..e259734e --- /dev/null +++ b/agent/pkg/manifest/manifest.go @@ -0,0 +1,19 @@ +package manifest + +import "time" + +// Manifest includes both user-defined file systems and system generated metadata. It is intended to +// help future proof the manifest definition by encapsulating user-defined filesystems so we can add +// system generated or other field as needed in a backwards compatible manner (e.g., versioning). +type Manifest struct { + Metadata Metadata `yaml:"metadata"` + Filesystems Filesystems `yaml:"filesystems"` +} + +// Metadata contains auto-generated fields that are appended to the active manifest. +type Metadata struct { + Updated time.Time `yaml:"updated"` +} + +// Filesystems is a map of FsUUIDs to file systems. +type Filesystems map[string]Filesystem diff --git a/agent/pkg/manifest/manifest.yaml b/agent/pkg/manifest/manifest.yaml new file mode 100644 index 00000000..b93e912b --- /dev/null +++ b/agent/pkg/manifest/manifest.yaml @@ -0,0 +1,77 @@ +filesystems: + 3b6f972b-64c7-4378-9f8e-172cf88c7d93: #fsUUID + common: + auth: + secret: "sharedSecret" + tls: + key: | + tlsKey + cert: | + tlsCert + config: + mgmtd: + beemsg-port: 10000 + meta: + connMetaPort: 0 + quotaEnableEnforcement: true + storeClientXAttrs: true + storeClientACLs: true + storage: + connStoragePort: 0 + quotaEnableEnforcement: true + client: + connClientPort: 0 + quotaEnabled: true + install-source: + type: package + repo: https://www.beegfs.io/release/beegfs_8.0/ + refs: + mgmtd: beegfs-mgmtd=8.0.1 + meta: beegfs-meta=8.0.1 + storage: beegfs-storage=8.0.1 + client: beegfs-client=8.0.1 + remote: beegfs-remote=8.0.1 + sync: beegfs-sync=8.0.1 + # source: + # type: container + # repo: ghcr.io/thinkparq + # refs: + # mgmtd: beegfs-mgmtd:8.0.1 + # meta: beegfs-meta:8.0.1 + # storage: beegfs-storage:8.0.1 + # client: beegfs-client:8.0.1 + # remote: beegfs-remote:8.0.1 + # sync: beegfs-sync:8.0.1 + agents: + agent1: # agentID + address: "127.0.0.1:9010" + interfaces: + - name: enp0s1 + address: "127.0.0.1/24" + services: + - type: mgmtd + targets: + - id: 101 + path: /beegfs/ + - type: meta + id: 1 + executable: /home/tux/development/beegfs/meta/build/beegfs-meta + interfaces: + - name: enp0s1 # IP configuration handled globally + targets: + - id: 101 + path: /beegfs/ + # ulfs: + # device: /dev/sda1 + # type: ext4 + # format_flags: foo + # mount_flags: baz + agent2: # agentID + services: + - type: storage + id: 1 + config: + tuneNumWorkers: 28 + agent3: # agentID + services: + - type: client diff --git a/agent/pkg/manifest/service.go b/agent/pkg/manifest/service.go new file mode 100644 index 00000000..622ed692 --- /dev/null +++ b/agent/pkg/manifest/service.go @@ -0,0 +1,58 @@ +package manifest + +import ( + "fmt" + "path/filepath" + + "github.com/thinkparq/beegfs-go/common/beegfs" +) + +type Service struct { + // shortUUID is set by InheritGlobalConfig and used internally to generate globally unique names + // and identifiers in case resources for multiple file systems exist on the same machine. + shortUUID string + longUUID string + ID beegfs.NumId `yaml:"id"` + Type beegfs.NodeType `yaml:"type"` + Config map[string]string `yaml:"config"` + Interfaces []Nic `yaml:"interfaces"` + Targets []Target `yaml:"targets"` + Executable string `yaml:"executable"` +} + +// GetTargetsConfig returns the string used to initialize +func (s Service) GetTargetsConfig() (map[string]string, error) { + switch s.Type { + case beegfs.Management: + if len(s.Targets) != 1 { + return nil, fmt.Errorf("invalid number of targets for node type %s: %d", s.Type.String(), len(s.Targets)) + } + path := filepath.Clean(s.Targets[0].GetPath() + "/mgmtd.sqlite") + return map[string]string{"db-file": path}, nil + default: + return nil, nil + } + + // TODO: Implement remaining node types. + // return "", nil, fmt.Errorf("unsupported node type: %v", s.Type) +} + +func (s Service) GetDescription() string { + return fmt.Sprintf("BeeGFS %s-%s-%d (managed by BeeOND)", s.shortUUID, s.Type, s.ID) +} + +func (s Service) GetSystemdUnit() string { + return fmt.Sprintf("%s-beegfs-%s-%d.service", s.shortUUID, s.Type, s.ID) +} + +func (s Service) GetConfig() []string { + config := make([]string, 0, len(s.Config)) + for k, v := range s.Config { + if s.Type == beegfs.Management { + config = append(config, fmt.Sprintf("--%s=%v", k, v)) + } else { + config = append(config, fmt.Sprintf("%s=%v", k, v)) + } + } + return config +} diff --git a/agent/pkg/manifest/target.go b/agent/pkg/manifest/target.go new file mode 100644 index 00000000..73008681 --- /dev/null +++ b/agent/pkg/manifest/target.go @@ -0,0 +1,104 @@ +package manifest + +import ( + "fmt" + "path" + "path/filepath" + "strings" + + "github.com/thinkparq/beegfs-go/common/beegfs" + pb "github.com/thinkparq/protobuf/go/agent" +) + +type Target struct { + // shortUUID is set by InheritGlobalConfig and used internally to generate globally unique names + // and identifiers in case resources for multiple file systems exist on the same machine. + shortUUID string + longUUID string + nodeType beegfs.NodeType + initCmd string + ID beegfs.NumId `yaml:"id"` + Path string `yaml:"path"` + ULFS *UnderlyingFS `yaml:"ulfs"` +} + +func (t Target) GetInitCmd() (string, []string) { + if t.nodeType == beegfs.Management { + return t.initCmd, []string{ + fmt.Sprintf("--fs-uuid=%s", t.longUUID), + fmt.Sprintf("--init"), + fmt.Sprintf("--db-file=%s", filepath.Clean(t.GetPath()+"/mgmtd.sqlite")), + } + } + // TODO: Setup init commands for other node types. + return t.initCmd, []string{} +} + +func (t Target) GetPath() string { + return path.Join(t.Path, t.shortUUID, fmt.Sprintf("%s_%d", t.nodeType, t.ID)) +} + +type UnderlyingFS struct { + Device string `yaml:"device"` + Type UnderlyingFSType `yaml:"type"` + FormatFlags string `yaml:"format_flags"` + MountFlags string `yaml:"mount_flags"` +} + +type UnderlyingFSType int + +const ( + UnknownUnderlyingFS UnderlyingFSType = iota + EXT4UnderlyingFS +) + +func (t UnderlyingFSType) String() string { + switch t { + case EXT4UnderlyingFS: + return "ext4" + default: + return "unknown" + } +} + +func (t *UnderlyingFSType) UnmarshalYAML(unmarshal func(any) error) error { + var s string + if err := unmarshal(&s); err != nil { + return err + } + + switch strings.ToLower(s) { + case "ext4": + *t = EXT4UnderlyingFS + default: + return fmt.Errorf("invalid underlying fs type: %s", s) + } + return nil +} + +func (t UnderlyingFSType) MarshalYAML() (any, error) { + switch t { + case EXT4UnderlyingFS: + return "ext4", nil + default: + return nil, fmt.Errorf("unknown fs type: %d", t) + } +} + +func ulfsTypeFromProto(fs pb.Target_UnderlyingFSOpts_FsType) UnderlyingFSType { + switch fs { + case pb.Target_UnderlyingFSOpts_EXT4: + return EXT4UnderlyingFS + default: + return UnknownUnderlyingFS + } +} + +func (fs UnderlyingFSType) toProto() pb.Target_UnderlyingFSOpts_FsType { + switch fs { + case EXT4UnderlyingFS: + return pb.Target_UnderlyingFSOpts_EXT4 + default: + return pb.Target_UnderlyingFSOpts_UNSPECIFIED + } +} diff --git a/agent/pkg/reconciler/errors.go b/agent/pkg/reconciler/errors.go new file mode 100644 index 00000000..f80e43c4 --- /dev/null +++ b/agent/pkg/reconciler/errors.go @@ -0,0 +1,9 @@ +package reconciler + +import "errors" + +var ( + ErrLoadingManifest = errors.New("unable to load manifest from disk") + ErrSavingManifest = errors.New("unable to save manifest to disk") + ErrBadManifest = errors.New("manifest failed verification") +) diff --git a/agent/pkg/reconciler/reconciler.go b/agent/pkg/reconciler/reconciler.go new file mode 100644 index 00000000..6361854b --- /dev/null +++ b/agent/pkg/reconciler/reconciler.go @@ -0,0 +1,249 @@ +package reconciler + +import ( + "context" + "errors" + "fmt" + "path" + "reflect" + "strings" + "sync" + "time" + + "github.com/google/uuid" + "github.com/thinkparq/beegfs-go/agent/pkg/deploy" + "github.com/thinkparq/beegfs-go/agent/pkg/manifest" + "github.com/thinkparq/beegfs-go/common/beegfs" + pb "github.com/thinkparq/protobuf/go/agent" + "go.uber.org/zap" +) + +type Config struct { + ManifestPath string `mapstructure:"manifest-path"` + ActiveManifestPath string `mapstructure:"active-manifest-path"` + DeploymentStrategy Strategy `mapstructure:"deployment-strategy"` +} + +type Strategy string + +const ( + DefaultStrategy Strategy = "default" +) + +type Configurer interface { + GetReconcilerConfig() Config +} + +// Reconciler is responsible for comparing the current/active Filesystem with the new desired +// Filesystem and decides what needs to be created, updated, or destroyed. +type Reconciler interface { + GetAgentID() string + GetFsUUID() string + Status() (ReconcileResult, error) + Cancel(string) (ReconcileResult, error) + UpdateConfiguration(any) error + Stop() error +} + +type ReconcileResult struct { + Status *pb.Status +} + +type defaultReconciler struct { + agentID string + log *zap.Logger + mu sync.Mutex + active manifest.Manifest + state state + config Config + strategy deploy.Deployer +} + +func New(ctx context.Context, agentID string, log *zap.Logger, config Config) (Reconciler, error) { + log = log.With(zap.String("component", path.Base(reflect.TypeOf(defaultReconciler{}).PkgPath()))) + var deploymentStrategy deploy.Deployer + var err error + switch config.DeploymentStrategy { + case DefaultStrategy: + if deploymentStrategy, err = deploy.NewDefaultStrategy(ctx); err != nil { + return nil, fmt.Errorf("unable to configure deployment strategy: %w", err) + } + default: + return nil, fmt.Errorf("unknown deployment strategy: %v", config.DeploymentStrategy) + } + // Setting the initial config and file system manifest will be triggered later by ConfigMgr. + return &defaultReconciler{ + agentID: agentID, + log: log, + state: newAgentState(log), + mu: sync.Mutex{}, + strategy: deploymentStrategy, + }, nil +} + +func (r *defaultReconciler) Stop() error { + r.log.Info("attempting to stop reconciler") + r.state.cancel("agent is shutting down") + r.mu.Lock() + defer r.mu.Unlock() + return r.strategy.Cleanup() +} + +func (r *defaultReconciler) GetAgentID() string { + return r.agentID +} + +func (r *defaultReconciler) GetFsUUID() string { + return "TODO" +} + +func (r *defaultReconciler) Status() (ReconcileResult, error) { + return ReconcileResult{ + Status: r.state.get(), + }, nil +} + +func (r *defaultReconciler) Cancel(reason string) (ReconcileResult, error) { + r.state.cancel(reason) + return r.Status() +} + +// UpdateConfiguration handles: +// +// - Local config updates from ConfigMgr where the new manifest is loaded from disk. +// - Remote config updates from the gRPC server where a new manifest is saved to disk. +// +// In both cases it will verify the new manifest and attempt to reconcile it if possible. +func (r *defaultReconciler) UpdateConfiguration(config any) error { + if configurer, ok := config.(Configurer); ok { + r.mu.Lock() + r.config = configurer.GetReconcilerConfig() + r.log.Info("loading file system manifest", zap.String("path", r.config.ManifestPath)) + newManifest, err := manifest.FromDisk(r.config.ManifestPath) + r.mu.Unlock() + if err != nil { + return fmt.Errorf("%w: %w", ErrLoadingManifest, err) + } + return r.verify(newManifest.Filesystems) + } else if newManifest, ok := config.(manifest.Manifest); ok { + r.mu.Lock() + r.log.Info("saving file system manifest", zap.String("path", r.config.ActiveManifestPath)) + err := manifest.ToDisk(newManifest, r.config.ManifestPath) + r.mu.Unlock() + if err != nil { + return fmt.Errorf("%w: %w", ErrSavingManifest, err) + } + return r.verify(newManifest.Filesystems) + } + return fmt.Errorf("%w: received unexpected manifest (most likely this indicates a bug and a report should be filed)", ErrBadManifest) +} + +// Verify performs any checks that can be done without actually reconciling the manifest. This +// allows a response to be returned quickly while the reconciliation happens in the background. +func (r *defaultReconciler) verify(newFilesystems manifest.Filesystems) error { + r.log.Info("verifying manifest") + if len(newFilesystems) == 0 { + return errors.New("manifest does not contain any file systems") + } + + // shortToFullUUIDs is used to ensure the short UUIDs derived from the full v4 FS UUID do not + // have any collisions. While true collisions are HIGHLY unlikely from properly generated v4 + // UUIDs, this might happen if there are user generated UUIDs, typos, or copy/paste errors. + shortToFullUUIDs := map[string]string{} + for fsUUID, fs := range newFilesystems { + fsUUID = strings.ToLower(fsUUID) + u, err := uuid.Parse(fsUUID) + if err != nil { + return fmt.Errorf("error parsing file system UUID: %w (is it a valid v4 UUID?)", err) + } else if u.Version() != 4 { + return fmt.Errorf("unsupported file system UUID version: %d (must be v4)", u.Version()) + } + shortUUID := manifest.ShortUUID(u) + if conflictingUUID, ok := shortToFullUUIDs[shortUUID]; ok { + return fmt.Errorf( + "short UUID collision: %q derived from %q and %q (first %d characters are identical)", + shortUUID, fsUUID, conflictingUUID, manifest.ShortUUIDLen, + ) + } + shortToFullUUIDs[shortUUID] = fsUUID + // TODO: + // * Avoid necessary reconciliations by seeing if the manifest changed. + // * Validate we can migrate from currentFS to newFS. + // * Validate the FS config: + // * All services have IPs + targets. + // * Services have the correct number of targets (i.e., 1 for mgmtd meta, remote, sync). + // Note these should be implemented as methods on manifest.Filesystem. + if err := fs.InheritGlobalConfig(shortUUID, fsUUID); err != nil { + return fmt.Errorf("error propagating global configuration: %w", err) + } + } + go r.reconcile(newFilesystems) + return nil +} + +// Reconcile attempts to move the local state from the currentFS to the newFS. +func (r *defaultReconciler) reconcile(newFilesystems manifest.Filesystems) { + r.mu.Lock() + defer r.mu.Unlock() + r.log.Debug("reconciling", zap.Any("filesystem", newFilesystems)) + ctx := r.state.start() + + for fsUUID, fs := range newFilesystems { + agent, ok := fs.Agents[r.agentID] + if !ok { + // Not all file systems in this manifest may have configuration for this agent. It is + // also valid that this manifest has no services managed by this agent. + r.log.Debug("file system has no services assigned to this agent", zap.String("fsUUID", fsUUID)) + continue + } + + // Don't apply any common configuration if the agent doesn't have any services for this file system. + if err := r.strategy.ApplySourceRepo(ctx, fs.Common.InstallSource); err != nil { + r.state.fail(fmt.Sprintf("unable to apply source configuration for %s: %s", fsUUID, err.Error())) + return + } + + if err := r.strategy.ApplyInterfaces(ctx, agent.Interfaces); err != nil { + r.state.fail(fmt.Sprintf("unable to apply global interface configuration for %s: %s", fsUUID, err.Error())) + return + } + + // Install packages for all services types managed by this agent: + installPackages := map[beegfs.NodeType]struct{}{} + for _, service := range agent.Services { + if _, ok := installPackages[service.Type]; ok { + continue + } + installPackages[service.Type] = struct{}{} + if ref, ok := fs.Common.InstallSource.Refs[service.Type]; ok { + if err := r.strategy.ApplyInstall(ctx, ref); err != nil { + r.state.fail(fmt.Sprintf("unable to apply service installation for %s: %s", getFsServiceID(fsUUID, service.Type, service.ID), err.Error())) + } + } + } + + // Roll out services: + for _, service := range agent.Services { + if err := r.strategy.ApplyInterfaces(ctx, service.Interfaces); err != nil { + r.state.fail(fmt.Sprintf("unable to apply interface configuration for %s: %s", getFsServiceID(fsUUID, service.Type, service.ID), err.Error())) + return + } + if err := r.strategy.ApplyTargets(ctx, service.Targets); err != nil { + r.state.fail(fmt.Sprintf("unable to apply target configuration for %s: %s", getFsServiceID(fsUUID, service.Type, service.ID), err.Error())) + return + } + if err := r.strategy.ApplyService(ctx, service); err != nil { + r.state.fail(fmt.Sprintf("unable to apply service configuration for %s: %s", getFsServiceID(fsUUID, service.Type, service.ID), err.Error())) + return + } + } + } + r.active = manifest.Manifest{ + Metadata: manifest.Metadata{ + Updated: time.Now(), + }, + Filesystems: newFilesystems, + } + manifest.ToDisk(r.active, r.config.ActiveManifestPath) + r.state.complete(pb.Status_SUCCESS) +} diff --git a/agent/pkg/reconciler/state.go b/agent/pkg/reconciler/state.go new file mode 100644 index 00000000..a99a2413 --- /dev/null +++ b/agent/pkg/reconciler/state.go @@ -0,0 +1,111 @@ +package reconciler + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/thinkparq/beegfs-go/common/beegfs" + pb "github.com/thinkparq/protobuf/go/agent" + "go.uber.org/zap" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type state struct { + logger *zap.Logger + current pb.Status + historical map[time.Time]*pb.Status + mu sync.Mutex + ctx context.Context + ctxCancel context.CancelFunc +} + +func newAgentState(l *zap.Logger) state { + // Always initialize with a valid context even though it is always replaced when starting a + // reconciliation. Otherwise stopping the reconciler would SIGSEGV when calling cancel. + ctx, cancel := context.WithCancel(context.Background()) + return state{ + current: pb.Status{ + State: pb.Status_IDLE, + Messages: []string{}, + Updated: timestamppb.Now(), + }, + historical: make(map[time.Time]*pb.Status), + mu: sync.Mutex{}, + logger: l, + ctx: ctx, + ctxCancel: cancel, + } +} + +func getFsServiceID(fsUUID string, nt beegfs.NodeType, id beegfs.NumId) string { + return fmt.Sprintf("%s:%s:%d", fsUUID, nt, id) +} + +// start() marks the beginning of a reconciliation. It returns a context that will be cancelled if +// the reconciliation is cancelled early. +func (s *state) start() context.Context { + s.mu.Lock() + defer s.mu.Unlock() + s.logger.Info("state update", zap.String("oldState", s.current.State.String()), zap.String("newState", pb.Status_APPLYING.String())) + s.historical[time.Now()] = proto.Clone(&s.current).(*pb.Status) + s.current = pb.Status{ + State: pb.Status_APPLYING, + Messages: []string{}, + Updated: timestamppb.Now(), + } + ctx, cancel := context.WithCancel(context.Background()) + s.ctx = ctx + s.ctxCancel = cancel + s.logUnlocked("began reconciliation") + return s.ctx +} + +func (s *state) get() *pb.Status { + s.mu.Lock() + defer s.mu.Unlock() + return proto.Clone(&s.current).(*pb.Status) +} + +func (s *state) logUnlocked(message string) { + s.current.Updated = timestamppb.Now() + s.current.Messages = append(s.current.Messages, fmt.Sprintf("%s: %s", s.current.Updated.String(), message)) +} + +func (s *state) log(message string) { + s.mu.Lock() + defer s.mu.Unlock() + s.current.Updated = timestamppb.Now() + s.current.Messages = append(s.current.Messages, fmt.Sprintf("%s: %s", s.current.Updated.String(), message)) +} + +func (s *state) fail(message string) *pb.Status { + s.mu.Lock() + defer s.mu.Unlock() + s.logger.Info("state update", zap.String("oldState", s.current.State.String()), zap.String("newState", pb.Status_FAILED.String()), zap.Any("message", message)) + s.current.State = pb.Status_FAILED + s.logUnlocked(message) + s.ctxCancel() + return proto.Clone(&s.current).(*pb.Status) +} + +func (s *state) cancel(message string) *pb.Status { + s.mu.Lock() + defer s.mu.Unlock() + s.logger.Info("state update", zap.String("oldState", s.current.State.String()), zap.String("newState", pb.Status_CANCELLED.String()), zap.Any("message", message)) + s.current.State = pb.Status_CANCELLED + s.logUnlocked(message) + s.ctxCancel() + return proto.Clone(&s.current).(*pb.Status) +} + +func (s *state) complete(finalState pb.Status_State) { + s.mu.Lock() + defer s.mu.Unlock() + s.logger.Info("state update", zap.String("oldState", s.current.State.String()), zap.String("newState", finalState.String())) + s.current.State = finalState + s.logUnlocked("finished reconciliation") + s.ctxCancel() +} diff --git a/common/beegfs/nodetype.go b/common/beegfs/nodetype.go index fec834be..d141d64b 100644 --- a/common/beegfs/nodetype.go +++ b/common/beegfs/nodetype.go @@ -1,6 +1,7 @@ package beegfs import ( + "fmt" "strings" pb "github.com/thinkparq/protobuf/go/beegfs" @@ -10,12 +11,15 @@ import ( // (which is technically correct, a meta target can only be on a meta server after all). type NodeType int +const InvalidNodeTypeString = "" const ( InvalidNodeType NodeType = iota Client Meta Storage Management + Remote + Sync ) // Create a NodeType from a string. Providing a non-ambiguous prefix is sufficient, e.g. for client, @@ -34,12 +38,19 @@ func NodeTypeFromString(input string) NodeType { return Management } + // To avoid ambiguity with storage, specifying sync requires at least 2 characters. + if len(input) >= 2 && (strings.HasPrefix("sync", input)) { + return Sync + } + if strings.HasPrefix("client", input) { return Client } else if strings.HasPrefix("storage", input) { return Storage } else if strings.HasPrefix("metadata", input) { return Meta + } else if strings.HasPrefix("remote", input) { + return Remote } return InvalidNodeType @@ -55,6 +66,10 @@ func NodeTypeFromProto(input pb.NodeType) NodeType { return Storage case pb.NodeType_MANAGEMENT: return Management + case pb.NodeType_REMOTE: + return Remote + case pb.NodeType_SYNC: + return Sync } return InvalidNodeType @@ -72,6 +87,10 @@ func (n NodeType) ToProto() *pb.NodeType { nt = pb.NodeType_STORAGE case Management: nt = pb.NodeType_MANAGEMENT + case Remote: + nt = pb.NodeType_REMOTE + case Sync: + nt = pb.NodeType_SYNC } return &nt @@ -88,7 +107,34 @@ func (n NodeType) String() string { return "storage" case Management: return "management" + case Remote: + return "remote" + case Sync: + return "sync" default: - return "" + return InvalidNodeTypeString + } +} + +func (n *NodeType) UnmarshalYAML(unmarshal func(any) error) error { + var s string + if err := unmarshal(&s); err != nil { + return err + } + + nodeType := NodeTypeFromString(s) + if nodeType == InvalidNodeType { + return fmt.Errorf("invalid node type: %q", s) + } + + *n = nodeType + return nil +} + +func (n NodeType) MarshalYAML() (any, error) { + str := n.String() + if str == InvalidNodeTypeString { + return nil, fmt.Errorf("cannot marshal invalid NodeType: %d", n) } + return str, nil } diff --git a/common/beegfs/nodetype_test.go b/common/beegfs/nodetype_test.go index bfb897af..a6131ffa 100644 --- a/common/beegfs/nodetype_test.go +++ b/common/beegfs/nodetype_test.go @@ -15,6 +15,10 @@ func TestFromString(t *testing.T) { assert.Equal(t, Client, NodeTypeFromString("c")) assert.Equal(t, Management, NodeTypeFromString(" management ")) assert.Equal(t, Management, NodeTypeFromString("ma")) + assert.Equal(t, Remote, NodeTypeFromString(" remote ")) + assert.Equal(t, Remote, NodeTypeFromString("r")) + assert.Equal(t, Sync, NodeTypeFromString(" sync ")) + assert.Equal(t, Sync, NodeTypeFromString("sy")) assert.Equal(t, InvalidNodeType, NodeTypeFromString("")) assert.Equal(t, InvalidNodeType, NodeTypeFromString("abc")) diff --git a/go.mod b/go.mod index 0ddf0d93..74f3a397 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/thinkparq/beegfs-go go 1.23.8 +replace github.com/thinkparq/protobuf => ../protobuf + require ( github.com/aws/aws-sdk-go-v2 v1.25.2 github.com/aws/aws-sdk-go-v2/config v1.27.6 @@ -9,6 +11,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.51.3 github.com/aws/smithy-go v1.20.4 github.com/bmatcuk/doublestar/v4 v4.8.1 + github.com/coreos/go-systemd/v22 v22.5.0 github.com/dgraph-io/badger/v4 v4.3.0 github.com/dsnet/golib/unitconv v1.0.2 github.com/google/uuid v1.6.0 @@ -28,6 +31,7 @@ require ( google.golang.org/grpc v1.71.1 google.golang.org/protobuf v1.36.6 gopkg.in/natefinch/lumberjack.v2 v2.2.1 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -49,6 +53,7 @@ require ( github.com/dgraph-io/ristretto v0.1.2-0.20240116140435-c67e07994f91 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/godbus/dbus/v5 v5.0.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect @@ -77,5 +82,4 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20250407143221-ac9807e6c755 // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index a8648034..882186f7 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -70,6 +72,8 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -169,8 +173,6 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/thinkparq/protobuf v0.8.1-0.20250602183119-1bcce2b457a2 h1:j5myww+83y4lGdENQzcrpR0uEAaKMiHbfQYAeazSJx4= -github.com/thinkparq/protobuf v0.8.1-0.20250602183119-1bcce2b457a2/go.mod h1:AaUUy9mWaja/EggLSfzbKydAe+We+440z/6FdmPz5yI= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=