From 1f79b9b33e93ed3d9ad3b85cac2cff9b0c28facb Mon Sep 17 00:00:00 2001 From: Anbraten Date: Fri, 28 Jul 2023 14:00:23 +0200 Subject: [PATCH 1/5] code cleanup --- .ecrc | 1 - cmd/woodpecker-autoscaler/flags.go | 21 +++++----- cmd/woodpecker-autoscaler/main.go | 14 ++++--- engine/autoscaler.go | 41 ++++++++----------- {drivers => providers}/hetznercloud/driver.go | 2 +- {drivers => providers}/hetznercloud/flags.go | 0 server/client.go | 26 ++++++------ 7 files changed, 47 insertions(+), 58 deletions(-) rename {drivers => providers}/hetznercloud/driver.go (99%) rename {drivers => providers}/hetznercloud/flags.go (100%) diff --git a/.ecrc b/.ecrc index 88a7b356..38430362 100644 --- a/.ecrc +++ b/.ecrc @@ -6,7 +6,6 @@ "vendor", "fixtures", "LICENSE", - "drivers/hetznercloud/driver.go", "_test.go", "Makefile" ] diff --git a/cmd/woodpecker-autoscaler/flags.go b/cmd/woodpecker-autoscaler/flags.go index 44af11ad..08f65fbd 100644 --- a/cmd/woodpecker-autoscaler/flags.go +++ b/cmd/woodpecker-autoscaler/flags.go @@ -6,8 +6,6 @@ import ( "github.com/urfave/cli/v2" ) -const optionIntervalDefault = "1m" - var flags = []cli.Flag{ &cli.StringFlag{ Name: "log-level", @@ -16,15 +14,16 @@ var flags = []cli.Flag{ EnvVars: []string{"WOODPECKER_LOG_LEVEL"}, }, &cli.StringFlag{ - Name: "interval", - Value: optionIntervalDefault, + Name: "reconciliation-interval", + Value: "1m", Usage: "reconciliation interval", - EnvVars: []string{"WOODPECKER_INTERVAL"}, + EnvVars: []string{"WOODPECKER_RECONCILIATION_INTERVAL"}, }, &cli.StringFlag{ - Name: "pool-id", - Value: "1", - Usage: "id of the scaler pool", + Name: "pool-id", + Value: "1", + Usage: "id of the autoscaler pool", + EnvVars: []string{"WOODPECKER_POOL_ID"}, }, &cli.IntFlag{ Name: "min-agents", @@ -45,13 +44,13 @@ var flags = []cli.Flag{ EnvVars: []string{"WOODPECKER_WORKFLOWS_PER_AGENT"}, }, &cli.StringFlag{ - Name: "server", + Name: "server-url", Value: "http://localhost:8000", Usage: "woodpecker server address", EnvVars: []string{"WOODPECKER_SERVER"}, }, &cli.StringFlag{ - Name: "token", + Name: "server-token", Usage: "woodpecker api token", EnvVars: []string{"WOODPECKER_TOKEN"}, FilePath: os.Getenv("WOODPECKER_TOKEN_FILE"), @@ -70,7 +69,7 @@ var flags = []cli.Flag{ }, &cli.StringFlag{ Name: "provider", - Value: "hetznercloud", + Value: "", Usage: "cloud provider to use", EnvVars: []string{"WOODPECKER_PROVIDER"}, }, diff --git a/cmd/woodpecker-autoscaler/main.go b/cmd/woodpecker-autoscaler/main.go index ac786043..757440b7 100644 --- a/cmd/woodpecker-autoscaler/main.go +++ b/cmd/woodpecker-autoscaler/main.go @@ -8,7 +8,7 @@ import ( "github.com/rs/zerolog" "github.com/rs/zerolog/log" - "github.com/woodpecker-ci/autoscaler/drivers/hetznercloud" + "github.com/woodpecker-ci/autoscaler/providers/hetznercloud" "github.com/woodpecker-ci/autoscaler/server" _ "github.com/joho/godotenv/autoload" @@ -22,13 +22,15 @@ func setupProvider(ctx *cli.Context, config *config.Config) (engine.Provider, er switch driver := ctx.String("provider"); driver { case "hetznercloud": return hetznercloud.New(ctx, config, driver) + case "": + return nil, fmt.Errorf("Please select a provider") } return nil, fmt.Errorf("unknown provider: %s", ctx.String("provider")) } func run(ctx *cli.Context) error { - log.Log().Msgf("Start autoscaler LogLevel = %s", zerolog.GlobalLevel().String()) + log.Log().Msgf("Starting autoscaler with log-level=%s", zerolog.GlobalLevel().String()) client, err := server.NewClient(ctx) if err != nil { @@ -62,18 +64,18 @@ func run(ctx *cli.Context) error { autoscaler := engine.NewAutoscaler(provider, client, config) - interval, err := time.ParseDuration(ctx.String("interval")) + reconciliationInterval, err := time.ParseDuration(ctx.String("reconciliation-interval")) if err != nil { - log.Error().Err(err).Msgf("cant parse reconciliation interval, use default: %v", optionIntervalDefault) - interval, _ = time.ParseDuration(optionIntervalDefault) + return fmt.Errorf("can't parse reconciliation interval %v", ctx.String("reconciliation-interval")) } for { select { case <-ctx.Done(): return nil - case <-time.After(interval): + case <-time.After(reconciliationInterval): if err := autoscaler.Reconcile(ctx.Context); err != nil { + log.Error().Err(err).Msg("draining agents failed") return err } } diff --git a/engine/autoscaler.go b/engine/autoscaler.go index 00a0feca..434c02ea 100644 --- a/engine/autoscaler.go +++ b/engine/autoscaler.go @@ -27,8 +27,7 @@ func NewAutoscaler(provider Provider, client woodpecker.Client, config *config.C } } -// nolint:revive -func (a *Autoscaler) getQueueInfo(ctx context.Context) (freeTasks, runningTasks, pendingTasks int, err error) { +func (a *Autoscaler) getQueueInfo(_ context.Context) (freeTasks, runningTasks, pendingTasks int, err error) { info, err := a.client.QueueInfo() if err != nil { return -1, -1, -1, fmt.Errorf("client.QueueInfo: %w", err) @@ -37,8 +36,7 @@ func (a *Autoscaler) getQueueInfo(ctx context.Context) (freeTasks, runningTasks, return info.Stats.Workers, info.Stats.Running, info.Stats.Pending + info.Stats.WaitingOnDeps, nil } -// nolint:revive -func (a *Autoscaler) loadAgents(ctx context.Context) error { +func (a *Autoscaler) loadAgents(_ context.Context) error { a.agents = []*woodpecker.Agent{} agents, err := a.client.AgentList() @@ -89,8 +87,7 @@ func (a *Autoscaler) createAgents(ctx context.Context, amount int) error { return nil } -// nolint:revive -func (a *Autoscaler) drainAgents(ctx context.Context, amount int) error { +func (a *Autoscaler) drainAgents(_ context.Context, amount int) error { for i := 0; i < amount; i++ { for _, agent := range a.agents { created := time.Unix(agent.Created, 0) @@ -112,7 +109,7 @@ func (a *Autoscaler) drainAgents(ctx context.Context, amount int) error { return nil } -func (a *Autoscaler) AgentIdle(agent *woodpecker.Agent) (bool, error) { +func (a *Autoscaler) isAgentIdle(agent *woodpecker.Agent) (bool, error) { tasks, err := a.client.AgentTasksList(agent.ID) if err != nil { return false, fmt.Errorf("client.AgentTasksList: %w", err) @@ -124,7 +121,7 @@ func (a *Autoscaler) AgentIdle(agent *woodpecker.Agent) (bool, error) { func (a *Autoscaler) removeDrainedAgents(ctx context.Context) error { for _, agent := range a.agents { if agent.NoSchedule { - isIdle, err := a.AgentIdle(agent) + isIdle, err := a.isAgentIdle(agent) if err != nil { return err } @@ -152,7 +149,7 @@ func (a *Autoscaler) removeDrainedAgents(ctx context.Context) error { return nil } -func (a *Autoscaler) removeDetachedAgents(ctx context.Context) error { +func (a *Autoscaler) cleanupAgents(ctx context.Context) error { registeredAgents := a.getPoolAgents(false) deployedAgentNames, err := a.provider.ListDeployedAgentNames(ctx) if err != nil { @@ -225,43 +222,37 @@ func (a *Autoscaler) calcAgents(ctx context.Context) (float64, error) { func (a *Autoscaler) Reconcile(ctx context.Context) error { if err := a.loadAgents(ctx); err != nil { - log.Error().Err(err).Msg("load agents failed") - return nil + return fmt.Errorf("loading agents failed: %w", err) } reqPoolAgents, err := a.calcAgents(ctx) if err != nil { - log.Error().Err(err).Msg("calculate agents failed") - return nil + return fmt.Errorf("calculating agents failed: %w", err) } if reqPoolAgents > 0 { - log.Debug().Msgf("start %v additional agents", reqPoolAgents) + log.Debug().Msgf("starting %f additional agents", reqPoolAgents) if err := a.createAgents(ctx, int(reqPoolAgents)); err != nil { - log.Error().Err(err).Msgf("create agents failed") - return nil + return fmt.Errorf("creating agents failed: %w", err) } } if reqPoolAgents < 0 { - num := int(math.Abs(float64(reqPoolAgents))) + num := int(math.Abs(reqPoolAgents)) - log.Debug().Msgf("try to stop %v agents", num) + log.Debug().Msgf("trying to stop %d agents", num) if err := a.drainAgents(ctx, num); err != nil { - log.Error().Err(err).Msg("remove agents failed") - return nil + return fmt.Errorf("draining agents failed: %w", err) } } - if err := a.removeDetachedAgents(ctx); err != nil { - log.Error().Err(err).Msg("remove agents failed") - return nil + if err := a.cleanupAgents(ctx); err != nil { + return fmt.Errorf("cleanup of agents failed: %w", err) } if err := a.removeDrainedAgents(ctx); err != nil { - log.Error().Err(err).Msg("remove agents failed") - return nil + return fmt.Errorf("removing drained agents failed: %w", err) } return nil diff --git a/drivers/hetznercloud/driver.go b/providers/hetznercloud/driver.go similarity index 99% rename from drivers/hetznercloud/driver.go rename to providers/hetznercloud/driver.go index e3e8b6f2..93495080 100644 --- a/drivers/hetznercloud/driver.go +++ b/providers/hetznercloud/driver.go @@ -103,7 +103,7 @@ func New(c *cli.Context, config *config.Config, name string) (engine.Provider, e Networks: c.StringSlice("hetznercloud-networks"), EnableIPv4: c.Bool("hetznercloud-public-ipv4-enable"), EnableIPv6: c.Bool("hetznercloud-public-ipv6-enable"), - LabelPrefix: "wp.scaler/", + LabelPrefix: "wp.autoscaler/", Config: config, } diff --git a/drivers/hetznercloud/flags.go b/providers/hetznercloud/flags.go similarity index 100% rename from drivers/hetznercloud/flags.go rename to providers/hetznercloud/flags.go diff --git a/server/client.go b/server/client.go index 1263a0d7..928e4ddc 100644 --- a/server/client.go +++ b/server/client.go @@ -17,21 +17,19 @@ import ( // NewClient returns a new client from the CLI context. func NewClient(c *cli.Context) (woodpecker.Client, error) { var ( - skip = c.Bool("skip-verify") - socks = c.String("socks-proxy") - socksoff = c.Bool("socks-proxy-off") - token = c.String("token") - server = c.String("server") + skip = c.Bool("skip-verify") + socks = c.String("socks-proxy") + socksoff = c.Bool("socks-proxy-off") + serverToken = c.String("server-token") + serverURL = c.String("server-url") ) - server = strings.TrimRight(server, "/") + serverURL = strings.TrimRight(serverURL, "/") - // if no server url is provided we can default - // to the hosted Woodpecker service. - if len(server) == 0 { - return nil, fmt.Errorf("Error: you must provide the Woodpecker server address") + if len(serverURL) == 0 { + return nil, fmt.Errorf("Error: Please provide the Woodpecker server address") } - if len(token) == 0 { - return nil, fmt.Errorf("Error: you must provide your Woodpecker access token") + if len(serverToken) == 0 { + return nil, fmt.Errorf("Error: Please provide a Woodpecker access token") } // attempt to find system CA certs @@ -48,7 +46,7 @@ func NewClient(c *cli.Context) (woodpecker.Client, error) { client := config.Client( c.Context, &oauth2.Token{ - AccessToken: token, + AccessToken: serverToken, }, ) @@ -71,5 +69,5 @@ func NewClient(c *cli.Context) (woodpecker.Client, error) { } } - return woodpecker.NewClient(server, client), nil + return woodpecker.NewClient(serverURL, client), nil } From 25ba2c4c530a825628421f6e56ea04908014eab0 Mon Sep 17 00:00:00 2001 From: Anbraten Date: Fri, 28 Jul 2023 17:41:01 +0200 Subject: [PATCH 2/5] rename --- engine/autoscaler.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/engine/autoscaler.go b/engine/autoscaler.go index 434c02ea..5f34c1df 100644 --- a/engine/autoscaler.go +++ b/engine/autoscaler.go @@ -74,7 +74,7 @@ func (a *Autoscaler) createAgents(ctx context.Context, amount int) error { return fmt.Errorf("client.AgentCreate: %w", err) } - log.Info().Str("agent", agent.Name).Msg("deploy agent") + log.Info().Str("agent", agent.Name).Msg("deploying agent") err = a.provider.DeployAgent(ctx, agent) if err != nil { @@ -89,13 +89,13 @@ func (a *Autoscaler) createAgents(ctx context.Context, amount int) error { func (a *Autoscaler) drainAgents(_ context.Context, amount int) error { for i := 0; i < amount; i++ { + now := time.Now() for _, agent := range a.agents { - created := time.Unix(agent.Created, 0) - duration, _ := time.ParseDuration("10m") - minAge := created.Add(duration) + agentCreate := time.Unix(agent.Created, 0) + minAgentAgentAlive, _ := time.ParseDuration("10m") - if !agent.NoSchedule && minAge.Unix() < time.Now().Unix() { - log.Info().Str("agent", agent.Name).Msg("drain agent") + if !agent.NoSchedule && agentCreate.Add(minAgentAgentAlive).Before(now) { + log.Info().Str("agent", agent.Name).Msg("draining agent") agent.NoSchedule = true _, err := a.client.AgentUpdate(agent) if err != nil { @@ -126,11 +126,11 @@ func (a *Autoscaler) removeDrainedAgents(ctx context.Context) error { return err } if !isIdle { - log.Info().Str("agent", agent.Name).Msg("agent is processing workload") + log.Info().Str("agent", agent.Name).Msg("agent is still processing workload") continue } - log.Info().Str("agent", agent.Name).Msgf("remove agent") + log.Info().Str("agent", agent.Name).Msgf("removing agent") err = a.provider.RemoveAgent(ctx, agent) if err != nil { From e5f3f53af92ab4bf93b22eed5624dac023c5e2d3 Mon Sep 17 00:00:00 2001 From: Anbraten Date: Fri, 28 Jul 2023 18:02:40 +0200 Subject: [PATCH 3/5] fix import --- cmd/woodpecker-autoscaler/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/woodpecker-autoscaler/main.go b/cmd/woodpecker-autoscaler/main.go index 7d7a561a..0951d659 100644 --- a/cmd/woodpecker-autoscaler/main.go +++ b/cmd/woodpecker-autoscaler/main.go @@ -8,7 +8,6 @@ import ( "github.com/rs/zerolog" "github.com/rs/zerolog/log" - "github.com/woodpecker-ci/autoscaler/drivers/hetznercloud" "github.com/woodpecker-ci/autoscaler/engine" "github.com/woodpecker-ci/autoscaler/providers/hetznercloud" "github.com/woodpecker-ci/autoscaler/server" From a748bc2279cfc1efb77c1729301125b5bc51a1f2 Mon Sep 17 00:00:00 2001 From: Anbraten Date: Fri, 28 Jul 2023 18:07:50 +0200 Subject: [PATCH 4/5] undo --- cmd/woodpecker-autoscaler/main.go | 5 +---- engine/autoscaler.go | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/cmd/woodpecker-autoscaler/main.go b/cmd/woodpecker-autoscaler/main.go index 0951d659..5a6e2b32 100644 --- a/cmd/woodpecker-autoscaler/main.go +++ b/cmd/woodpecker-autoscaler/main.go @@ -79,10 +79,7 @@ func run(ctx *cli.Context) error { case <-ctx.Done(): return nil case <-time.After(reconciliationInterval): - if err := autoscaler.Reconcile(ctx.Context); err != nil { - log.Error().Err(err).Msg("draining agents failed") - return err - } + autoscaler.Reconcile(ctx.Context) } } } diff --git a/engine/autoscaler.go b/engine/autoscaler.go index d1055679..7584fc2f 100644 --- a/engine/autoscaler.go +++ b/engine/autoscaler.go @@ -220,21 +220,24 @@ func (a *Autoscaler) calcAgents(ctx context.Context) (float64, error) { return reqPoolAgents, nil } -func (a *Autoscaler) Reconcile(ctx context.Context) error { +func (a *Autoscaler) Reconcile(ctx context.Context) { if err := a.loadAgents(ctx); err != nil { - return fmt.Errorf("loading agents failed: %w", err) + log.Error().Err(err).Msg("load agents failed") + return } reqPoolAgents, err := a.calcAgents(ctx) if err != nil { - return fmt.Errorf("calculating agents failed: %w", err) + log.Error().Err(err).Msg("calculating agents failed") + return } if reqPoolAgents > 0 { log.Debug().Msgf("starting %f additional agents", reqPoolAgents) if err := a.createAgents(ctx, int(reqPoolAgents)); err != nil { - return fmt.Errorf("creating agents failed: %w", err) + log.Error().Err(err).Msg("creating agents failed") + return } } @@ -243,17 +246,18 @@ func (a *Autoscaler) Reconcile(ctx context.Context) error { log.Debug().Msgf("trying to stop %d agents", num) if err := a.drainAgents(ctx, num); err != nil { - return fmt.Errorf("draining agents failed: %w", err) + log.Error().Err(err).Msg("draining agents failed") + return } } if err := a.cleanupAgents(ctx); err != nil { - return fmt.Errorf("cleanup of agents failed: %w", err) + log.Error().Err(err).Msg("cleanup of agents failed") + return } if err := a.removeDrainedAgents(ctx); err != nil { - return fmt.Errorf("removing drained agents failed: %w", err) + log.Error().Err(err).Msg("removing drained agents failed") + return } - - return nil } From 64720de026295f2b95e90e3ca32769e970ca3706 Mon Sep 17 00:00:00 2001 From: Anbraten Date: Fri, 28 Jul 2023 18:12:40 +0200 Subject: [PATCH 5/5] adjust --- .ecrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.ecrc b/.ecrc index 38430362..23f3bc42 100644 --- a/.ecrc +++ b/.ecrc @@ -6,6 +6,7 @@ "vendor", "fixtures", "LICENSE", + "providers/hetznercloud/driver.go", "_test.go", "Makefile" ]