From 8dc99a194c27471689e67e312c71a8a6c1d9a68d Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Fri, 16 Jun 2023 08:54:09 +0200 Subject: [PATCH] Support reload using signal Signed-off-by: Simon Pasquier --- go.mod | 1 + go.sum | 2 + pkg/reloader/reloader.go | 116 ++++++++++++++++++++++++++++++++++----- 3 files changed, 104 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 44c7e822251..ad6a1b43717 100644 --- a/go.mod +++ b/go.mod @@ -118,6 +118,7 @@ require ( ) require ( + github.com/mitchellh/go-ps v1.0.0 github.com/onsi/gomega v1.27.6 go.opentelemetry.io/contrib/propagators/autoprop v0.38.0 go4.org/intern v0.0.0-20220617035311-6925f38cc365 diff --git a/go.sum b/go.sum index d801f1f5851..392be70da72 100644 --- a/go.sum +++ b/go.sum @@ -674,6 +674,8 @@ github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= diff --git a/pkg/reloader/reloader.go b/pkg/reloader/reloader.go index 1072026c2ac..3ac603d3699 100644 --- a/pkg/reloader/reloader.go +++ b/pkg/reloader/reloader.go @@ -56,6 +56,7 @@ import ( "bytes" "compress/gzip" "context" + "fmt" "hash" "io" "net/http" @@ -66,12 +67,14 @@ import ( "regexp" "strings" "sync" + "syscall" "time" "github.com/fsnotify/fsnotify" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/minio/sha256-simd" + ps "github.com/mitchellh/go-ps" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -84,8 +87,6 @@ import ( // Referenced environment variables must be of the form `$(var)` (not `$var` or `${var}`). type Reloader struct { logger log.Logger - reloadURL *url.URL - httpClient http.Client cfgFile string cfgOutputFile string watchInterval time.Duration @@ -93,6 +94,8 @@ type Reloader struct { watchedDirs []string watcher *watcher + tr TriggerReloader + lastCfgHash []byte lastWatchedDirsHash []byte forceReload bool @@ -105,10 +108,21 @@ type Reloader struct { configApply prometheus.Counter } +// TriggerReloader reloads the configuration of the process. +type TriggerReloader interface { + TriggerReload(ctx context.Context) error +} + // Options bundles options for the Reloader. type Options struct { - // ReloadURL is a prometheus URL to trigger reloads. + // ReloadURL is the Prometheus URL to trigger reloads. ReloadURL *url.URL + // HTTP client used to connect to the web server. + HTTPClient http.Client + // ProcessName is the process executable name to trigger reloads. If not + // empty, the reloader sends a SIGHUP signal to the matching process ID + // instead of using the HTTP reload endpoint. + ProcessName string // CfgFile is a path to the prometheus config file to watch. CfgFile string // CfgOutputFile is a path for the output config file. @@ -124,7 +138,7 @@ type Options struct { // WatchInterval controls how often reloader re-reads config and directories. WatchInterval time.Duration // RetryInterval controls how often the reloader retries a reloading of the - // configuration in case the endpoint returned an error. + // configuration in case the reload operation returned an error. RetryInterval time.Duration } @@ -138,7 +152,6 @@ func New(logger log.Logger, reg prometheus.Registerer, o *Options) *Reloader { } r := &Reloader{ logger: logger, - reloadURL: o.ReloadURL, cfgFile: o.CfgFile, cfgOutputFile: o.CfgOutputFile, watcher: newWatcher(logger, reg, o.DelayInterval), @@ -183,6 +196,13 @@ func New(logger log.Logger, reg prometheus.Registerer, o *Options) *Reloader { }, ), } + + if o.ProcessName != "" { + r.tr = NewPIDReloader(o.ProcessName) + } else { + r.tr = NewHTTPReloader(r.logger, o.ReloadURL, o.HTTPClient) + } + return r } @@ -201,6 +221,12 @@ func (r *Reloader) Watch(ctx context.Context) error { return nil } + if _, ok := r.tr.(*PIDReloader); ok { + level.Info(r.logger).Log("msg", "reloading via process signal") + } else { + level.Info(r.logger).Log("msg", "reloading via HTTP") + } + defer runutil.CloseWithLogOnErr(r.logger, r.watcher, "config watcher close") if r.cfgFile != "" { @@ -361,8 +387,9 @@ func (r *Reloader) apply(ctx context.Context) error { if r.watchInterval == 0 { return nil } + r.reloads.Inc() - if err := r.triggerReload(ctx); err != nil { + if err := r.tr.TriggerReload(ctx); err != nil { r.reloadErrors.Inc() r.lastReloadSuccess.Set(0) return errors.Wrap(err, "trigger reload") @@ -410,28 +437,87 @@ func hashFile(h hash.Hash, fn string) error { return nil } -func (r *Reloader) triggerReload(ctx context.Context) error { - req, err := http.NewRequest("POST", r.reloadURL.String(), nil) +type PIDReloader struct { + pname string +} + +func NewPIDReloader(pname string) *PIDReloader { + return &PIDReloader{ + pname: pname, + } +} + +func (pr *PIDReloader) TriggerReload(ctx context.Context) error { + procs, err := ps.Processes() + if err != nil { + return fmt.Errorf("list processes: %w", err) + } + + var proc ps.Process + for i := range procs { + if pr.pname == procs[i].Executable() { + proc = procs[i] + break + } + } + + if proc == nil { + return fmt.Errorf("failed to find process matching %q", pr.pname) + } + + p, err := os.FindProcess(proc.Pid()) + if err != nil { + return fmt.Errorf("find process err: %w", err) + } + + if proc == nil { + return fmt.Errorf("failed to find process with pid %d", proc.Pid()) + } + + if err := p.Signal(syscall.SIGHUP); err != nil { + return fmt.Errorf("failed to send SIGHUP to pid %d: %w", p.Pid, err) + } + + return nil +} + +var _ = TriggerReloader(&PIDReloader{}) + +type HTTPReloader struct { + logger log.Logger + + u *url.URL + c http.Client +} + +var _ = TriggerReloader(&HTTPReloader{}) + +func NewHTTPReloader(logger log.Logger, u *url.URL, c http.Client) *HTTPReloader { + return &HTTPReloader{ + logger: logger, + u: u, + c: c, + } +} + +func (hr *HTTPReloader) TriggerReload(ctx context.Context) error { + req, err := http.NewRequest("POST", hr.u.String(), nil) if err != nil { return errors.Wrap(err, "create request") } req = req.WithContext(ctx) - resp, err := r.httpClient.Do(req) + resp, err := hr.c.Do(req) if err != nil { return errors.Wrap(err, "reload request failed") } - defer runutil.ExhaustCloseWithLogOnErr(r.logger, resp.Body, "trigger reload resp body") + defer runutil.ExhaustCloseWithLogOnErr(hr.logger, resp.Body, "trigger reload resp body") if resp.StatusCode != 200 { return errors.Errorf("received non-200 response: %s; have you set `--web.enable-lifecycle` Prometheus flag?", resp.Status) } - return nil -} -// SetHttpClient sets Http client for reloader. -func (r *Reloader) SetHttpClient(client http.Client) { - r.httpClient = client + return nil } // ReloadURLFromBase returns the standard Prometheus reload URL from its base URL.