From a12d296e6f0c434f42114189e0d5c245755979ad Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Wed, 28 Dec 2022 13:36:08 +0100 Subject: [PATCH] Jitter watch option --- docs/config/index.md | 3 +++ docs/config/watch.md | 19 +++++++++++++++++++ docs/install/docker.md | 3 +++ docs/providers/docker.md | 1 + docs/providers/kubernetes.md | 2 ++ docs/providers/swarm.md | 1 + docs/usage/basic-example.md | 1 + docs/user-guides/docker-file-providers.md | 1 + go.mod | 2 +- go.sum | 4 ++-- internal/app/diun.go | 4 ++-- internal/config/config_test.go | 1 + internal/config/fixtures/config.test.yml | 1 + internal/config/fixtures/config.validate.yml | 1 + internal/model/watch.go | 14 +++++++++----- test/docker2/diun.yml | 1 + test/docker3/diun.yml | 1 + 17 files changed, 50 insertions(+), 10 deletions(-) diff --git a/docs/config/index.md b/docs/config/index.md index 28d75b1a..26342998 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -32,6 +32,7 @@ You can override this using the [`--config` flag or `CONFIG` env var with `serve watch: workers: 10 schedule: "0 */6 * * *" + jitter: 30s firstCheckNotif: false notif: @@ -121,6 +122,7 @@ All configuration from file can be transposed into environment variables. As an watch: workers: 10 schedule: "0 */6 * * *" + jitter: 30s firstCheckNotif: false notif: @@ -169,6 +171,7 @@ Can be transposed to: DIUN_WATCH_WORKERS=10 DIUN_WATCH_SCHEDULE=0 */6 * * * + DIUN_WATCH_JITTER=30s DIUN_WATCH_FIRSTCHECKNOTIF=false DIUN_NOTIF_GOTIFY_ENDPOINT=http://gotify.foo.com diff --git a/docs/config/watch.md b/docs/config/watch.md index b6809d07..bf8d2687 100644 --- a/docs/config/watch.md +++ b/docs/config/watch.md @@ -6,6 +6,7 @@ watch: workers: 10 schedule: "0 */6 * * *" + jitter: 30s firstCheckNotif: false compareDigest: true healthchecks: @@ -44,6 +45,24 @@ Maximum number of workers that will execute tasks concurrently. (default `10`) !!! abstract "Environment variables" * `DIUN_WATCH_SCHEDULE` +### `jitter` + +Enable time jitter. Prior to executing of a job, cron will sleep a random +duration in the range from 0 to _jitter_. (default `30s`) + +!!! note + Only works with `schedule` setting. `0` disables time jitter. + +!!! example "Config file" + ```yaml + watch: + schedule: "0 */6 * * *" + jitter: 30s + ``` + +!!! abstract "Environment variables" + * `DIUN_WATCH_JITTER` + ### `firstCheckNotif` Send notification at the very first analysis of an image. (default `false`) diff --git a/docs/install/docker.md b/docs/install/docker.md index 72eb8837..2cad9264 100644 --- a/docs/install/docker.md +++ b/docs/install/docker.md @@ -60,6 +60,7 @@ services: - "LOG_JSON=false" - "DIUN_WATCH_WORKERS=20" - "DIUN_WATCH_SCHEDULE=0 */6 * * *" + - "DIUN_WATCH_JITTER=30s" - "DIUN_PROVIDERS_DOCKER=true" labels: - "diun.enable=true" @@ -82,6 +83,7 @@ docker run -d --name diun \ -e "LOG_JSON=false" \ -e "DIUN_WATCH_WORKERS=20" \ -e "DIUN_WATCH_SCHEDULE=0 */6 * * *" \ + -e "DIUN_WATCH_JITTER=30s" \ -e "DIUN_PROVIDERS_DOCKER=true" \ -v "$(pwd)/data:/data" \ -v "/var/run/docker.sock:/var/run/docker.sock" \ @@ -153,6 +155,7 @@ services: - "LOG_JSON=false" - "DIUN_WATCH_WORKERS=20" - "DIUN_WATCH_SCHEDULE=0 */6 * * *" + - "DIUN_WATCH_JITTER=30s" - "DIUN_PROVIDERS_DOCKER=true" labels: - "diun.enable=true" diff --git a/docs/providers/docker.md b/docs/providers/docker.md index 800f038d..91460b4c 100644 --- a/docs/providers/docker.md +++ b/docs/providers/docker.md @@ -31,6 +31,7 @@ services: - "LOG_JSON=false" - "DIUN_WATCH_WORKERS=20" - "DIUN_WATCH_SCHEDULE=0 */6 * * *" + - "DIUN_WATCH_JITTER=30s" - "DIUN_PROVIDERS_DOCKER=true" restart: always diff --git a/docs/providers/kubernetes.md b/docs/providers/kubernetes.md index 5a7670bd..5ca1a31c 100644 --- a/docs/providers/kubernetes.md +++ b/docs/providers/kubernetes.md @@ -82,6 +82,8 @@ spec: value: "20" - name: DIUN_WATCH_SCHEDULE value: "0 */6 * * *" + - name: DIUN_WATCH_JITTER + value: "30s" - name: DIUN_PROVIDERS_KUBERNETES value: "true" volumeMounts: diff --git a/docs/providers/swarm.md b/docs/providers/swarm.md index b6c2a0cf..fb62e128 100644 --- a/docs/providers/swarm.md +++ b/docs/providers/swarm.md @@ -30,6 +30,7 @@ services: - "LOG_JSON=false" - "DIUN_WATCH_WORKERS=20" - "DIUN_WATCH_SCHEDULE=0 */6 * * *" + - "DIUN_WATCH_JITTER=30s" - "DIUN_PROVIDERS_SWARM=true" deploy: mode: replicated diff --git a/docs/usage/basic-example.md b/docs/usage/basic-example.md index b1718bd6..730c4508 100644 --- a/docs/usage/basic-example.md +++ b/docs/usage/basic-example.md @@ -23,6 +23,7 @@ services: - "LOG_JSON=false" - "DIUN_WATCH_WORKERS=20" - "DIUN_WATCH_SCHEDULE=0 */6 * * *" + - "DIUN_WATCH_JITTER=30s" - "DIUN_PROVIDERS_DOCKER=true" - "DIUN_PROVIDERS_DOCKER_WATCHBYDEFAULT=true" restart: always diff --git a/docs/user-guides/docker-file-providers.md b/docs/user-guides/docker-file-providers.md index 77a13691..fe31dbb0 100644 --- a/docs/user-guides/docker-file-providers.md +++ b/docs/user-guides/docker-file-providers.md @@ -28,6 +28,7 @@ services: - "LOG_JSON=false" - "DIUN_WATCH_WORKERS=20" - "DIUN_WATCH_SCHEDULE=0 */6 * * *" + - "DIUN_WATCH_JITTER=30s" - "DIUN_PROVIDERS_DOCKER=true" - "DIUN_PROVIDERS_DOCKER_WATCHBYDEFAULT=true" - "DIUN_PROVIDERS_FILE_FILENAME=/custom-images.yml" diff --git a/go.mod b/go.mod index bd0ca3dc..185f2ce3 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/bmatcuk/doublestar/v3 v3.0.0 github.com/containerd/containerd v1.6.14 github.com/containers/image/v5 v5.23.1 + github.com/crazy-max/cron/v3 v3.1.1 github.com/crazy-max/gohealthchecks v0.3.0 github.com/crazy-max/gonfig v0.6.0 github.com/docker/docker v20.10.18+incompatible @@ -32,7 +33,6 @@ require ( github.com/panjf2000/ants/v2 v2.7.1 github.com/pkg/errors v0.9.1 github.com/pkg/profile v1.7.0 - github.com/robfig/cron/v3 v3.0.1 github.com/rs/zerolog v1.28.0 github.com/russross/blackfriday/v2 v2.1.0 github.com/sirupsen/logrus v1.9.0 diff --git a/go.sum b/go.sum index ef9d5ca0..b0552549 100644 --- a/go.sum +++ b/go.sum @@ -265,6 +265,8 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/crazy-max/cron/v3 v3.1.1 h1:8tsAXjV522daYSaN6/Mb/Nh8X/Ez+nedU0KuPV98rNU= +github.com/crazy-max/cron/v3 v3.1.1/go.mod h1:yexi3kKoh3GcnmRCppyJKsvYIBWzDVaym0dwNWo+zdg= github.com/crazy-max/gohealthchecks v0.3.0 h1:mXtTsOinEOcvRcnT3ayRkxGdK18LfrFOWE/gFpfzEI4= github.com/crazy-max/gohealthchecks v0.3.0/go.mod h1:gkT8QSdEXZJahyswdTGDbd+q20fWm0DmWW7TWBNtgJg= github.com/crazy-max/gonfig v0.6.0 h1:CwP+x1gfor4Ze8lLQKkZw/DOQEMVgwBcoocTYLo3c4A= @@ -762,8 +764,6 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= -github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= diff --git a/internal/app/diun.go b/internal/app/diun.go index 68fd1da4..850787d1 100644 --- a/internal/app/diun.go +++ b/internal/app/diun.go @@ -6,6 +6,7 @@ import ( "sync/atomic" "time" + "github.com/crazy-max/cron/v3" "github.com/crazy-max/diun/v4/internal/config" "github.com/crazy-max/diun/v4/internal/db" "github.com/crazy-max/diun/v4/internal/grpc" @@ -22,7 +23,6 @@ import ( "github.com/hako/durafmt" "github.com/panjf2000/ants/v2" "github.com/pkg/errors" - "github.com/robfig/cron/v3" "github.com/rs/zerolog/log" ) @@ -110,7 +110,7 @@ func (di *Diun) Start() error { if len(di.cfg.Watch.Schedule) == 0 { return nil } - di.jobID, err = di.cron.AddJob(di.cfg.Watch.Schedule, di) + di.jobID, err = di.cron.AddJobWithJitter(di.cfg.Watch.Schedule, di, *di.cfg.Watch.Jitter) if err != nil { return err } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 1d69e2b2..b4f7de77 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -52,6 +52,7 @@ func TestLoadFile(t *testing.T) { Watch: &model.Watch{ Workers: 100, Schedule: "*/30 * * * *", + Jitter: utl.NewDuration(30 * time.Second), FirstCheckNotif: utl.NewTrue(), CompareDigest: utl.NewTrue(), Healthchecks: &model.Healthchecks{ diff --git a/internal/config/fixtures/config.test.yml b/internal/config/fixtures/config.test.yml index 149b4949..2fb4758d 100644 --- a/internal/config/fixtures/config.test.yml +++ b/internal/config/fixtures/config.test.yml @@ -4,6 +4,7 @@ db: watch: workers: 100 schedule: "*/30 * * * *" + jitter: 30s firstCheckNotif: true compareDigest: true healthchecks: diff --git a/internal/config/fixtures/config.validate.yml b/internal/config/fixtures/config.validate.yml index b8dbd305..8a582427 100644 --- a/internal/config/fixtures/config.validate.yml +++ b/internal/config/fixtures/config.validate.yml @@ -4,6 +4,7 @@ db: watch: workers: 100 schedule: "*/30 * * * *" + jitter: 30s firstCheckNotif: false compareDigest: true healthchecks: diff --git a/internal/model/watch.go b/internal/model/watch.go index 5f98f043..05c1557e 100644 --- a/internal/model/watch.go +++ b/internal/model/watch.go @@ -1,16 +1,19 @@ package model import ( + "time" + "github.com/crazy-max/diun/v4/pkg/utl" ) // Watch holds data necessary for watch configuration type Watch struct { - Workers int `yaml:"workers,omitempty" json:"workers,omitempty" validate:"required,min=1"` - Schedule string `yaml:"schedule,omitempty" json:"schedule,omitempty"` - FirstCheckNotif *bool `yaml:"firstCheckNotif,omitempty" json:"firstCheckNotif,omitempty" validate:"required"` - CompareDigest *bool `yaml:"compareDigest,omitempty" json:"compareDigest,omitempty" validate:"required"` - Healthchecks *Healthchecks `yaml:"healthchecks,omitempty" json:"healthchecks,omitempty"` + Workers int `yaml:"workers,omitempty" json:"workers,omitempty" validate:"required,min=1"` + Schedule string `yaml:"schedule,omitempty" json:"schedule,omitempty"` + Jitter *time.Duration `yaml:"jitter,omitempty" json:"jitter,omitempty" validate:"required"` + FirstCheckNotif *bool `yaml:"firstCheckNotif,omitempty" json:"firstCheckNotif,omitempty" validate:"required"` + CompareDigest *bool `yaml:"compareDigest,omitempty" json:"compareDigest,omitempty" validate:"required"` + Healthchecks *Healthchecks `yaml:"healthchecks,omitempty" json:"healthchecks,omitempty"` } // GetDefaults gets the default values @@ -23,6 +26,7 @@ func (s *Watch) GetDefaults() *Watch { // SetDefaults sets the default values func (s *Watch) SetDefaults() { s.Workers = 10 + s.Jitter = utl.NewDuration(30 * time.Second) s.FirstCheckNotif = utl.NewFalse() s.CompareDigest = utl.NewTrue() } diff --git a/test/docker2/diun.yml b/test/docker2/diun.yml index 85e51152..f1807a5d 100644 --- a/test/docker2/diun.yml +++ b/test/docker2/diun.yml @@ -1,6 +1,7 @@ watch: workers: 20 schedule: "0 */6 * * *" + jitter: 5m providers: docker: diff --git a/test/docker3/diun.yml b/test/docker3/diun.yml index d7186e28..69856a0b 100644 --- a/test/docker3/diun.yml +++ b/test/docker3/diun.yml @@ -1,6 +1,7 @@ watch: workers: 20 schedule: "0 */6 * * *" + jitter: 0 providers: docker: