From 3410424cdc0bf36b6ffd2cb30858f384b4dc3e87 Mon Sep 17 00:00:00 2001 From: Martin Helmich Date: Thu, 30 Jul 2020 11:24:56 +0200 Subject: [PATCH] Support for "boot" jobs that are run to completion before "regular" jobs --- README.md | 15 +++++++ cmd/renderfiles.go | 5 ++- cmd/up.go | 17 +++++--- examples/mittnite.d/jobs/sleep.hcl | 5 +++ internal/config/ignitionconfig.go | 4 ++ internal/config/types.go | 20 +++++++-- pkg/proc/bootjob.go | 55 +++++++++++++++++++++++ pkg/proc/runner.go | 70 +++++++++++++++++++++++++----- pkg/proc/types.go | 29 ++++++++++++- 9 files changed, 197 insertions(+), 23 deletions(-) create mode 100644 pkg/proc/bootjob.go diff --git a/README.md b/README.md index 3b5db09..a36b08e 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,21 @@ If there is a `lazy` block present, the process itself will only be started when When no connections have been active for a duration of at least `.lazy.coolDownTimeout`, mittnite will terminate the process again. +#### Boot Jobs + +Boot jobs are "special" jobs that are executed before regular `job` definitions. Boot jobs are required to run to completion before any regular jobs are started. + +```hcl +boot "setup" { + command = "/bin/bash" + args = ["/init-script.sh"] + timeout = "30s" + env = [ + "FOO=bar" + ] +} +``` + #### File Possible directives to use in a file definition. diff --git a/cmd/renderfiles.go b/cmd/renderfiles.go index 346eec8..859e127 100644 --- a/cmd/renderfiles.go +++ b/cmd/renderfiles.go @@ -13,8 +13,11 @@ func init() { rootCmd.AddCommand(renderFiles) } +// TODO(@hermsi1337): WTH do we even need this for!? var renderFiles = &cobra.Command{ - Use: "renderfiles", + Use: "renderfiles", + Short: "Renders configuration files", + Long: "This command renders the configured configuration files, before (optionally) starting another process", Run: func(cmd *cobra.Command, args []string) { ignitionConfig := &config.Ignition{ Probes: nil, diff --git a/cmd/up.go b/cmd/up.go index e3030d0..9a40aca 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -19,7 +19,9 @@ func init() { } var up = &cobra.Command{ - Use: "up", + Use: "up", + Short: "Render config files, start probes and processes", + Long: "This sub-command renders the configuration files, starts the probes and launches all configured processes", Run: func(cmd *cobra.Command, args []string) { ignitionConfig := &config.Ignition{ Probes: nil, @@ -75,15 +77,20 @@ var up = &cobra.Command{ ctx, cancel := context.WithCancel(context.Background()) defer cancel() - runner := proc.NewRunner(ctx, ignitionConfig) + runner := proc.NewRunner(ignitionConfig) go func() { <-procSignals cancel() }() - err = runner.Run() - if err != nil { - log.Fatalf("service runner stopped with error: %s", err) + if err := runner.Boot(ctx); err != nil { + log.WithError(err).Fatal("runner error'ed during initialization") + } else { + log.Info("initialization complete") + } + + if err := runner.Run(ctx); err != nil { + log.WithError(err).Fatal("service runner stopped with error") } else { log.Print("service runner stopped without error") } diff --git a/examples/mittnite.d/jobs/sleep.hcl b/examples/mittnite.d/jobs/sleep.hcl index 26ec983..62222e2 100644 --- a/examples/mittnite.d/jobs/sleep.hcl +++ b/examples/mittnite.d/jobs/sleep.hcl @@ -3,4 +3,9 @@ job sleep { args = ["500s"] canFail = true oneTime = true +} + +boot hello { + command = "cowsay" + args = ["moo"] } \ No newline at end of file diff --git a/internal/config/ignitionconfig.go b/internal/config/ignitionconfig.go index 0d24785..cc889c2 100644 --- a/internal/config/ignitionconfig.go +++ b/internal/config/ignitionconfig.go @@ -34,6 +34,10 @@ func (ignitionConfig *Ignition) GenerateFromConfigDir(configDir string) error { } for _, job := range ignitionConfig.Jobs { + if job.OneTime { + log.Warnf("field oneTime in job %s is deprecated in favor of 'bootJob' directies", job.Name) + } + if job.MaxAttempts_ != 0 { log.Warnf("field max_attempts in job %s is deprecated in favor of maxAttempts", job.Name) job.MaxAttempts = job.MaxAttempts_ diff --git a/internal/config/types.go b/internal/config/types.go index 0b76c70..adaae65 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -87,7 +87,18 @@ type JobConfig struct { Laziness *Laziness `hcl:"lazy"` Listeners []Listener `hcl:"listen"` CanFail bool `hcl:"canFail"` - OneTime bool `hcl:"oneTime"` + + // DEPRECATED: Use "BootJob"s instead + OneTime bool `hcl:"oneTime"` +} + +type BootJobConfig struct { + Name string `hcl:",key"` + Command string `hcl:"command"` + Args []string `hcl:"args"` + Env []string `hcl:"env"` + CanFail bool `hcl:"canFail"` + Timeout string `hcl:"timeout"` } type File struct { @@ -98,7 +109,8 @@ type File struct { } type Ignition struct { - Probes []Probe `hcl:"probe"` - Files []File `hcl:"file"` - Jobs []JobConfig `hcl:"job"` + Probes []Probe `hcl:"probe"` + Files []File `hcl:"file"` + Jobs []JobConfig `hcl:"job"` + BootJobs []BootJobConfig `hcl:"boot"` } diff --git a/pkg/proc/bootjob.go b/pkg/proc/bootjob.go new file mode 100644 index 0000000..2a62ded --- /dev/null +++ b/pkg/proc/bootjob.go @@ -0,0 +1,55 @@ +package proc + +import ( + "context" + "fmt" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "os" + "os/exec" +) + +func (job *BootJob) Run(ctx context.Context) error { + l := log.WithField("job.name", job.Config.Name) + + ctx, job.cancelProcess = context.WithCancel(ctx) + + job.cmd = exec.CommandContext(ctx, job.Config.Command, job.Config.Args...) + job.cmd.Stdout = os.Stdout + job.cmd.Stderr = os.Stderr + if job.Config.Env != nil { + job.cmd.Env = append(os.Environ(), job.Config.Env...) + } + + l.Info("starting boot job") + + err := job.cmd.Start() + if err != nil { + return fmt.Errorf("failed to start job %s: %s", job.Config.Name, err.Error()) + } + + job.process = job.cmd.Process + + err = job.cmd.Wait() + if err != nil { + l.WithError(err).Error("job exited with error") + } else { + l.Info("boot job completed") + } + + if ctx.Err() != nil { // execution cancelled + return ctx.Err() + } + + if err != nil { + if job.Config.CanFail { + l.WithError(err).Warn("job failed, but is allowed to fail") + return nil + } + + l.WithError(err).Error("boot job failed") + return errors.Wrapf(err, "error while exec'ing boot job '%s'", job.Config.Name) + } + + return nil +} diff --git a/pkg/proc/runner.go b/pkg/proc/runner.go index 813ae2f..ff47375 100644 --- a/pkg/proc/runner.go +++ b/pkg/proc/runner.go @@ -9,14 +9,66 @@ import ( log "github.com/sirupsen/logrus" ) -func NewRunner(ctx context.Context, ignitionConfig *config.Ignition) *Runner { +func NewRunner(ignitionConfig *config.Ignition) *Runner { return &Runner{ IgnitionConfig: ignitionConfig, - jobs: []*Job{}, - ctx: ctx, + jobs: make([]*Job, 0, len(ignitionConfig.Jobs)), + bootJobs: make([]*BootJob, 0, len(ignitionConfig.BootJobs)), } } +func waitGroupToChannel(wg *sync.WaitGroup) <-chan struct{} { + d := make(chan struct{}) + go func() { + wg.Wait() + close(d) + }() + + return d +} + +func (r *Runner) Boot(ctx context.Context) error { + wg := sync.WaitGroup{} + errs := make(chan error) + + for j := range r.IgnitionConfig.BootJobs { + job, err := NewBootJob(&r.IgnitionConfig.BootJobs[j]) + if err != nil { + return err + } + + r.bootJobs = append(r.bootJobs, job) + } + + for _, job := range r.bootJobs { + wg.Add(1) + go func(job *BootJob) { + defer wg.Done() + + if err := job.Run(ctx); err != nil { + errs <- err + } + }(job) + } + + select { + case <-waitGroupToChannel(&wg): + return nil + + case <-ctx.Done(): + log.Warn("context cancelled") + return ctx.Err() + + case err, ok := <-errs: + if ok && err != nil { + log.Error("job return error, shutting down other services") + return err + } + } + + return nil +} + func (r *Runner) exec(ctx context.Context, wg *sync.WaitGroup, errChan chan<- error) { for j := range r.IgnitionConfig.Jobs { job, err := NewJob(&r.IgnitionConfig.Jobs[j]) @@ -42,24 +94,18 @@ func (r *Runner) exec(ctx context.Context, wg *sync.WaitGroup, errChan chan<- er } } -func (r *Runner) Run() error { +func (r *Runner) Run(ctx context.Context) error { errChan := make(chan error) ticker := time.NewTicker(5 * time.Second) wg := sync.WaitGroup{} - r.exec(r.ctx, &wg, errChan) - - allDone := make(chan struct{}) - go func() { - wg.Wait() - close(allDone) - }() + r.exec(ctx, &wg, errChan) for { select { // wait for them all to finish, or one to fail - case <-allDone: + case <-waitGroupToChannel(&wg): return nil // watch files diff --git a/pkg/proc/types.go b/pkg/proc/types.go index 7f6c499..3375f34 100644 --- a/pkg/proc/types.go +++ b/pkg/proc/types.go @@ -13,7 +13,15 @@ import ( type Runner struct { IgnitionConfig *config.Ignition jobs []*Job - ctx context.Context + bootJobs []*BootJob +} + +type BootJob struct { + Config *config.BootJobConfig + cmd *exec.Cmd + process *os.Process + timeout time.Duration + cancelProcess context.CancelFunc } type Job struct { @@ -63,3 +71,22 @@ func NewJob(c *config.JobConfig) (*Job, error) { return &j, nil } + +func NewBootJob(c *config.BootJobConfig) (*BootJob, error) { + bj := BootJob{ + Config: c, + } + + if ts := c.Timeout; ts != "" { + t, err := time.ParseDuration(ts) + if err != nil { + return nil, err + } + + bj.timeout = t + } else { + bj.timeout = 30 * time.Second + } + + return &bj, nil +}