From 1c4306c3fb54e4c7cf75b8f9821f439935fda197 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 17 Aug 2023 21:41:22 +0200 Subject: [PATCH 1/9] local backend: clone: check if scm is git first --- pipeline/backend/local/clone.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pipeline/backend/local/clone.go b/pipeline/backend/local/clone.go index e859eb8989..47b115254d 100644 --- a/pipeline/backend/local/clone.go +++ b/pipeline/backend/local/clone.go @@ -28,7 +28,6 @@ import ( "github.com/rs/zerolog/log" "github.com/woodpecker-ci/woodpecker/pipeline/backend/types" - "github.com/woodpecker-ci/woodpecker/shared/constant" ) // checkGitCloneCap check if we have the git binary on hand @@ -64,17 +63,20 @@ func (e *local) setupClone(state *workflowState) error { // execClone executes a clone-step locally func (e *local) execClone(ctx context.Context, step *types.Step, state *workflowState, env []string) error { - if err := e.setupClone(state); err != nil { - return fmt.Errorf("setup clone step failed: %w", err) + if scm := step.Environment["CI_REPO_SCM"]; scm != "git" { + return fmt.Errorf("local backend can only clone from git repos") } if err := checkGitCloneCap(); err != nil { return fmt.Errorf("check for git clone capabilities failed: %w", err) } - if step.Image != constant.DefaultCloneImage { - // TODO: write message into log - log.Warn().Msgf("clone step image '%s' does not match default git clone image. We ignore it assume git.", step.Image) + if err := e.setupClone(state); err != nil { + return fmt.Errorf("setup clone step failed: %w", err) + } + + if !strings.Contains(step.Image, "woodpeckerci/plugin-git") { + log.Warn().Msgf("clone step image '%s' does not match default git clone image. We ignore it and use our plugin-git anyway.", step.Image) } rmCmd, err := writeNetRC(step, state) From 28524405de4de97f76a369be66c2389ed57a4089 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 17 Aug 2023 21:51:45 +0200 Subject: [PATCH 2/9] add support for "plugins" on local backend --- pipeline/backend/local/local.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/pipeline/backend/local/local.go b/pipeline/backend/local/local.go index 88050bc5fc..581be6d076 100644 --- a/pipeline/backend/local/local.go +++ b/pipeline/backend/local/local.go @@ -120,13 +120,16 @@ func (e *local) StartStep(ctx context.Context, step *types.Step, taskUUID string return e.execClone(ctx, step, state, env) case types.StepTypeCommands: return e.execCommands(ctx, step, state, env) + case types.StepTypePlugin: + return e.execPlugin(ctx, step, state, env) default: return ErrUnsupportedStepType } } +// execCommands use step.Image as shell and run the commands in it func (e *local) execCommands(ctx context.Context, step *types.Step, state *workflowState, env []string) error { - // TODO: use commands directly + // TODO: find a way to simulate commands to be exec as stdin user commands instead of generating a script and hope the shell understands script := "" for _, cmd := range step.Commands { script += fmt.Sprintf("echo + %s\n%s\n", strings.TrimSpace(shellescape.Quote(cmd)), cmd) @@ -148,6 +151,22 @@ func (e *local) execCommands(ctx context.Context, step *types.Step, state *workf return cmd.Start() } +// execPlugin use step.Image as exec binary +func (e *local) execPlugin(ctx context.Context, step *types.Step, state *workflowState, env []string) error { + // Use "image name" as run command + cmd := exec.CommandContext(ctx, step.Image) + cmd.Env = env + cmd.Dir = state.workspaceDir + + // Get output and redirect Stderr to Stdout + e.output, _ = cmd.StdoutPipe() + cmd.Stderr = cmd.Stdout + + state.stepCMDs[step.Name] = cmd + + return cmd.Start() +} + // WaitStep for the pipeline step to complete and returns // the completion results. func (e *local) WaitStep(_ context.Context, step *types.Step, taskUUID string) (*types.State, error) { From 0ee356751bc778a86491cbee5cfbc2d0d949b643 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 17 Aug 2023 21:56:38 +0200 Subject: [PATCH 3/9] rm redundant comment --- pipeline/backend/local/local.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pipeline/backend/local/local.go b/pipeline/backend/local/local.go index 581be6d076..ba29978fde 100644 --- a/pipeline/backend/local/local.go +++ b/pipeline/backend/local/local.go @@ -153,7 +153,6 @@ func (e *local) execCommands(ctx context.Context, step *types.Step, state *workf // execPlugin use step.Image as exec binary func (e *local) execPlugin(ctx context.Context, step *types.Step, state *workflowState, env []string) error { - // Use "image name" as run command cmd := exec.CommandContext(ctx, step.Image) cmd.Env = env cmd.Dir = state.workspaceDir From e45c53a051004d4ce1e9769465e1c35a9820de7e Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 17 Aug 2023 22:55:13 +0200 Subject: [PATCH 4/9] docs: local backend: address todo as we have v1.0.x releases available --- docs/docs/30-administration/22-backends/20-local.md | 6 +----- .../version-1.0/30-administration/22-backends/20-local.md | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/docs/docs/30-administration/22-backends/20-local.md b/docs/docs/30-administration/22-backends/20-local.md index 5e57b04ce4..c8c5dbc4eb 100644 --- a/docs/docs/30-administration/22-backends/20-local.md +++ b/docs/docs/30-administration/22-backends/20-local.md @@ -6,11 +6,7 @@ The local backend will execute the pipelines on the local system without any iso :::note This backend is still pretty new and can not be treated as stable. Its -implementation and configuration can change at any time. Binary releases of the -agent will be available with the release of the [1.0.0 -milestone](https://github.com/woodpecker-ci/woodpecker/milestone/4), so for now -you must compile the agent by yourself, to get the local backend functionality. - +implementation and configuration can change at any time. ::: Since the code runs directly in the same context as the agent (same user, same diff --git a/docs/versioned_docs/version-1.0/30-administration/22-backends/20-local.md b/docs/versioned_docs/version-1.0/30-administration/22-backends/20-local.md index 5e57b04ce4..c8c5dbc4eb 100644 --- a/docs/versioned_docs/version-1.0/30-administration/22-backends/20-local.md +++ b/docs/versioned_docs/version-1.0/30-administration/22-backends/20-local.md @@ -6,11 +6,7 @@ The local backend will execute the pipelines on the local system without any iso :::note This backend is still pretty new and can not be treated as stable. Its -implementation and configuration can change at any time. Binary releases of the -agent will be available with the release of the [1.0.0 -milestone](https://github.com/woodpecker-ci/woodpecker/milestone/4), so for now -you must compile the agent by yourself, to get the local backend functionality. - +implementation and configuration can change at any time. ::: Since the code runs directly in the same context as the agent (same user, same From 991bf8463463681e14bece16a84fbf9d14e38297 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 17 Aug 2023 23:00:39 +0200 Subject: [PATCH 5/9] update docs acordingly --- docs/docs/30-administration/22-backends/20-local.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/docs/30-administration/22-backends/20-local.md b/docs/docs/30-administration/22-backends/20-local.md index c8c5dbc4eb..32eacf39e1 100644 --- a/docs/docs/30-administration/22-backends/20-local.md +++ b/docs/docs/30-administration/22-backends/20-local.md @@ -88,6 +88,17 @@ steps: [...] ``` +### Plugins are just binary's to be exec + +```yaml +steps: + build: + image: /usr/bin/tree +``` + +If there are no commands, we thread them as plugin as usual. +Plugins in local backend terms are just binary's to be executed by name if they can be looked up in `$PATH` or as absolute path. + ### Using labels to filter tasks You can use the [agent configuration From ff75008f2bb5a9f58f9c25a3c583c968ac86569d Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 17 Aug 2023 23:14:19 +0200 Subject: [PATCH 6/9] make explicite lookup and improve docs --- docs/docs/30-administration/22-backends/20-local.md | 6 +++--- pipeline/backend/local/local.go | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/docs/30-administration/22-backends/20-local.md b/docs/docs/30-administration/22-backends/20-local.md index 32eacf39e1..b03dd0bc11 100644 --- a/docs/docs/30-administration/22-backends/20-local.md +++ b/docs/docs/30-administration/22-backends/20-local.md @@ -88,7 +88,7 @@ steps: [...] ``` -### Plugins are just binary's to be exec +### Plugins as Executable Binaries ```yaml steps: @@ -96,8 +96,8 @@ steps: image: /usr/bin/tree ``` -If there are no commands, we thread them as plugin as usual. -Plugins in local backend terms are just binary's to be executed by name if they can be looked up in `$PATH` or as absolute path. +If no commands are provided, we treat them as plugins in the usual manner. +In the context of the local backend, plugins are simply executable binaries, which can be located using their name if they are listed in `$PATH`, or through an absolute path. ### Using labels to filter tasks diff --git a/pipeline/backend/local/local.go b/pipeline/backend/local/local.go index ba29978fde..2516ee9eb2 100644 --- a/pipeline/backend/local/local.go +++ b/pipeline/backend/local/local.go @@ -153,7 +153,12 @@ func (e *local) execCommands(ctx context.Context, step *types.Step, state *workf // execPlugin use step.Image as exec binary func (e *local) execPlugin(ctx context.Context, step *types.Step, state *workflowState, env []string) error { - cmd := exec.CommandContext(ctx, step.Image) + binary, err := exec.LookPath(step.Image) + if err != nil { + return fmt.Errorf("exec plugin: %w", err) + } + + cmd := exec.CommandContext(ctx, binary) cmd.Env = env cmd.Dir = state.workspaceDir From 0559827af636e5ffb68d1fe28202c15bc45e36a5 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 17 Aug 2023 23:16:32 +0200 Subject: [PATCH 7/9] Update pipeline/backend/local/clone.go --- pipeline/backend/local/clone.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/backend/local/clone.go b/pipeline/backend/local/clone.go index 47b115254d..bce46ead7d 100644 --- a/pipeline/backend/local/clone.go +++ b/pipeline/backend/local/clone.go @@ -64,7 +64,7 @@ func (e *local) setupClone(state *workflowState) error { // execClone executes a clone-step locally func (e *local) execClone(ctx context.Context, step *types.Step, state *workflowState, env []string) error { if scm := step.Environment["CI_REPO_SCM"]; scm != "git" { - return fmt.Errorf("local backend can only clone from git repos") + return fmt.Errorf("local backend can only clone from git repos, but this repo use '%s'", scm) } if err := checkGitCloneCap(); err != nil { From b84d0e3c64041f5311b81e67c98220fdb645cf56 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Thu, 17 Aug 2023 23:18:12 +0200 Subject: [PATCH 8/9] Update pipeline/backend/local/local.go --- pipeline/backend/local/local.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/backend/local/local.go b/pipeline/backend/local/local.go index 2516ee9eb2..998405711a 100644 --- a/pipeline/backend/local/local.go +++ b/pipeline/backend/local/local.go @@ -155,7 +155,7 @@ func (e *local) execCommands(ctx context.Context, step *types.Step, state *workf func (e *local) execPlugin(ctx context.Context, step *types.Step, state *workflowState, env []string) error { binary, err := exec.LookPath(step.Image) if err != nil { - return fmt.Errorf("exec plugin: %w", err) + return fmt.Errorf("lookup plugin binary: %w", err) } cmd := exec.CommandContext(ctx, binary) From eb5873dd18e512dfe559b5aa7333b59d49ae4eb7 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 18 Aug 2023 01:22:53 +0200 Subject: [PATCH 9/9] Update pipeline/backend/local/clone.go --- pipeline/backend/local/clone.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/backend/local/clone.go b/pipeline/backend/local/clone.go index bce46ead7d..019c9ec2ff 100644 --- a/pipeline/backend/local/clone.go +++ b/pipeline/backend/local/clone.go @@ -75,7 +75,7 @@ func (e *local) execClone(ctx context.Context, step *types.Step, state *workflow return fmt.Errorf("setup clone step failed: %w", err) } - if !strings.Contains(step.Image, "woodpeckerci/plugin-git") { + if !strings.Contains(step.Image, "plugin-git") { log.Warn().Msgf("clone step image '%s' does not match default git clone image. We ignore it and use our plugin-git anyway.", step.Image) }