diff --git a/.golangci.yml b/.golangci.yml index 0e15e61450..6c314d8e57 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -16,6 +16,92 @@ linters-settings: - ^log.Fatal().*$ errorlint: errorf-multi: true + depguard: + rules: + pipeline: + list-mode: lax + files: + - '**/pipeline/**/*.go' + - '**/pipeline/*.go' + - '!**/cli/pipeline/*.go' + - '!**/cli/pipeline/**/*.go' + - '!**/server/pipeline/*.go' + - '!**/server/pipeline/**/*.go' + deny: + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/agent' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/cli' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/cmd' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/server' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/web' + shared: + list-mode: lax + files: + - '**/shared/**/*.go' + - '**/shared/*.go' + - '!**/pipeline/shared/*.go' + - '!**/pipeline/shared/**/*.go' + deny: + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/agent' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/cli' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/cmd' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/pipeline' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/server' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/web' + woodpecker-go: + list-mode: lax + files: + - '**/woodpecker-go/woodpecker/**/*.go' + - '**/woodpecker-go/woodpecker/*.go' + deny: + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/agent' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/cli' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/cmd' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/pipeline' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/server' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/shared' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/web' + agent: + list-mode: lax + files: + - '**/agent/**/*.go' + - '**/agent/*.go' + - '**/cmd/agent/**/*.go' + - '**/cmd/agent/*.go' + deny: + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/cli' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/cmd/cli' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/cmd/server' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/server' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/web' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/woodpecker-go/woodpecker' + cli: + list-mode: lax + files: + - '**/cli/**/*.go' + - '**/cli/*.go' + - '**/cmd/cli/**/*.go' + - '**/cmd/cli/*.go' + deny: + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/agent' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/server' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/cmd/agent' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/cmd/server' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/web' + server: + list-mode: lax + files: + - '**/server/**/*.go' + - '**/server/*.go' + - '**/cmd/server/**/*.go' + - '**/cmd/server/*.go' + - '**/web/**/*.go' + - '**/web/*.go' + deny: + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/agent' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/cli' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/cmd/agent' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/cmd/cli' + - pkg: 'go.woodpecker-ci.org/woodpecker/v2/woodpecker-go/woodpecker' linters: disable-all: true @@ -37,9 +123,10 @@ linters: - errorlint - forbidigo - zerologlint + - depguard run: - timeout: 5m + timeout: 15m issues: exclude-rules: diff --git a/Makefile b/Makefile index 114ae51edc..3b7eedc790 100644 --- a/Makefile +++ b/Makefile @@ -141,7 +141,7 @@ ui-dependencies: ## Install UI dependencies .PHONY: lint lint: install-tools ## Lint code @echo "Running golangci-lint" - golangci-lint run --timeout 15m + golangci-lint run lint-ui: ## Lint UI code (cd web/; pnpm install) diff --git a/cli/exec/exec.go b/cli/exec/exec.go index 7e8c7fc668..3527530349 100644 --- a/cli/exec/exec.go +++ b/cli/exec/exec.go @@ -94,7 +94,7 @@ func runExec(c *cli.Context, file, repoPath string) error { axes, err := matrix.ParseString(string(dat)) if err != nil { - return fmt.Errorf("Parse matrix fail") + return fmt.Errorf("parse matrix fail") } if len(axes) == 0 { diff --git a/pipeline/frontend/metadata/subsitution.go b/pipeline/frontend/metadata/subsitution.go new file mode 100644 index 0000000000..1ceed7386c --- /dev/null +++ b/pipeline/frontend/metadata/subsitution.go @@ -0,0 +1,32 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadata + +import ( + "fmt" + "strings" + + "github.com/drone/envsubst" +) + +func EnvVarSubst(yaml string, environ map[string]string) (string, error) { + return envsubst.Eval(yaml, func(name string) string { + env := environ[name] + if strings.Contains(env, "\n") { + env = fmt.Sprintf("%q", env) + } + return env + }) +} diff --git a/pipeline/frontend/metadata/subsitution_test.go b/pipeline/frontend/metadata/subsitution_test.go new file mode 100644 index 0000000000..34b2c5d8f1 --- /dev/null +++ b/pipeline/frontend/metadata/subsitution_test.go @@ -0,0 +1,47 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metadata + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEnvVarSubst(t *testing.T) { + testCases := []struct { + name string + yaml string + environ map[string]string + want string + }{{ + name: "simple substitution", + yaml: `steps: + step1: + image: ${HELLO_IMAGE}`, + environ: map[string]string{"HELLO_IMAGE": "hello-world"}, + want: `steps: + step1: + image: hello-world`, + }} + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + result, err := EnvVarSubst(testCase.yaml, testCase.environ) + assert.NoError(t, err) + assert.EqualValues(t, testCase.want, result) + }) + } +} diff --git a/pipeline/frontend/yaml/constraint/constraint_test.go b/pipeline/frontend/yaml/constraint/constraint_test.go index 267fb4556f..3b0fa24a44 100644 --- a/pipeline/frontend/yaml/constraint/constraint_test.go +++ b/pipeline/frontend/yaml/constraint/constraint_test.go @@ -20,7 +20,6 @@ import ( "github.com/stretchr/testify/assert" "gopkg.in/yaml.v3" - "go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend" "go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata" ) @@ -543,7 +542,7 @@ func TestConstraints(t *testing.T) { for _, test := range testdata { t.Run(test.desc, func(t *testing.T) { - conf, err := frontend.EnvVarSubst(test.conf, test.with.Environ()) + conf, err := metadata.EnvVarSubst(test.conf, test.with.Environ()) assert.NoError(t, err) c := parseConstraints(t, conf) got, err := c.Match(test.with, false, test.env) diff --git a/server/pipeline/config.go b/server/pipeline/config.go index f592de973f..80ba15f9fe 100644 --- a/server/pipeline/config.go +++ b/server/pipeline/config.go @@ -18,9 +18,9 @@ import ( "crypto/sha256" "fmt" - "go.woodpecker-ci.org/woodpecker/v2/pipeline" forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/pipeline/stepbuilder" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) @@ -32,7 +32,7 @@ func findOrPersistPipelineConfig(store store.Store, currentPipeline *model.Pipel RepoID: currentPipeline.RepoID, Data: forgeYamlConfig.Data, Hash: sha, - Name: pipeline.SanitizePath(forgeYamlConfig.Name), + Name: stepbuilder.SanitizePath(forgeYamlConfig.Name), } err = store.ConfigCreate(conf) if err != nil { diff --git a/server/pipeline/items.go b/server/pipeline/items.go index a51a1ee7a6..d28b563388 100644 --- a/server/pipeline/items.go +++ b/server/pipeline/items.go @@ -21,16 +21,16 @@ import ( "github.com/rs/zerolog/log" - "go.woodpecker-ci.org/woodpecker/v2/pipeline" pipeline_errors "go.woodpecker-ci.org/woodpecker/v2/pipeline/errors" "go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/compiler" "go.woodpecker-ci.org/woodpecker/v2/server" forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/pipeline/stepbuilder" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) -func parsePipeline(store store.Store, currentPipeline *model.Pipeline, user *model.User, repo *model.Repo, yamls []*forge_types.FileMeta, envs map[string]string) ([]*pipeline.Item, error) { +func parsePipeline(store store.Store, currentPipeline *model.Pipeline, user *model.User, repo *model.Repo, yamls []*forge_types.FileMeta, envs map[string]string) ([]*stepbuilder.Item, error) { netrc, err := server.Config.Services.Forge.Netrc(user, repo) if err != nil { log.Error().Err(err).Msg("Failed to generate netrc file") @@ -66,7 +66,7 @@ func parsePipeline(store store.Store, currentPipeline *model.Pipeline, user *mod envs[k] = v } - b := pipeline.StepBuilder{ + b := stepbuilder.StepBuilder{ Repo: repo, Curr: currentPipeline, Last: last, @@ -89,7 +89,7 @@ func parsePipeline(store store.Store, currentPipeline *model.Pipeline, user *mod func createPipelineItems(c context.Context, store store.Store, currentPipeline *model.Pipeline, user *model.User, repo *model.Repo, yamls []*forge_types.FileMeta, envs map[string]string, -) (*model.Pipeline, []*pipeline.Item, error) { +) (*model.Pipeline, []*stepbuilder.Item, error) { pipelineItems, err := parsePipeline(store, currentPipeline, user, repo, yamls, envs) if pipeline_errors.HasBlockingErrors(err) { currentPipeline, uerr := UpdateToStatusError(store, *currentPipeline, err) @@ -113,7 +113,7 @@ func createPipelineItems(c context.Context, store store.Store, // setPipelineStepsOnPipeline is the link between pipeline representation in "pipeline package" and server // to be specific this func currently is used to convert the pipeline.Item list (crafted by StepBuilder.Build()) into // a pipeline that can be stored in the database by the server -func setPipelineStepsOnPipeline(pipeline *model.Pipeline, pipelineItems []*pipeline.Item) *model.Pipeline { +func setPipelineStepsOnPipeline(pipeline *model.Pipeline, pipelineItems []*stepbuilder.Item) *model.Pipeline { var pidSequence int for _, item := range pipelineItems { if pidSequence < item.Workflow.PID { diff --git a/server/pipeline/items_test.go b/server/pipeline/items_test.go index 3b418eb7a7..bec0c23372 100644 --- a/server/pipeline/items_test.go +++ b/server/pipeline/items_test.go @@ -3,9 +3,9 @@ package pipeline import ( "testing" - sharedPipeline "go.woodpecker-ci.org/woodpecker/v2/pipeline" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" + sharedPipeline "go.woodpecker-ci.org/woodpecker/v2/server/pipeline/stepbuilder" ) func TestSetPipelineStepsOnPipeline(t *testing.T) { diff --git a/server/pipeline/queue.go b/server/pipeline/queue.go index 657355d320..d40d0437ad 100644 --- a/server/pipeline/queue.go +++ b/server/pipeline/queue.go @@ -19,13 +19,13 @@ import ( "encoding/json" "fmt" - "go.woodpecker-ci.org/woodpecker/v2/pipeline" "go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc" "go.woodpecker-ci.org/woodpecker/v2/server" "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/pipeline/stepbuilder" ) -func queuePipeline(repo *model.Repo, pipelineItems []*pipeline.Item) error { +func queuePipeline(repo *model.Repo, pipelineItems []*stepbuilder.Item) error { var tasks []*model.Task for _, item := range pipelineItems { if item.Workflow.State == model.StatusSkipped { @@ -53,7 +53,7 @@ func queuePipeline(repo *model.Repo, pipelineItems []*pipeline.Item) error { return server.Config.Services.Queue.PushAtOnce(context.Background(), tasks) } -func taskIds(dependsOn []string, pipelineItems []*pipeline.Item) (taskIds []string) { +func taskIds(dependsOn []string, pipelineItems []*stepbuilder.Item) (taskIds []string) { for _, dep := range dependsOn { for _, pipelineItem := range pipelineItems { if pipelineItem.Workflow.Name == dep { diff --git a/server/pipeline/start.go b/server/pipeline/start.go index 99d1d6a562..42d1e64056 100644 --- a/server/pipeline/start.go +++ b/server/pipeline/start.go @@ -19,13 +19,13 @@ import ( "github.com/rs/zerolog/log" - "go.woodpecker-ci.org/woodpecker/v2/pipeline" "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/pipeline/stepbuilder" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) // start a pipeline, make sure it was stored persistent in the store before -func start(ctx context.Context, store store.Store, activePipeline *model.Pipeline, user *model.User, repo *model.Repo, pipelineItems []*pipeline.Item) (*model.Pipeline, error) { +func start(ctx context.Context, store store.Store, activePipeline *model.Pipeline, user *model.User, repo *model.Repo, pipelineItems []*stepbuilder.Item) (*model.Pipeline, error) { // call to cancel previous pipelines if needed if err := cancelPreviousPipelines(ctx, store, activePipeline, repo, user); err != nil { // should be not breaking diff --git a/pipeline/frontend/metadata.go b/server/pipeline/stepbuilder/metadata.go similarity index 92% rename from pipeline/frontend/metadata.go rename to server/pipeline/stepbuilder/metadata.go index 34134e03c0..63643b6a0c 100644 --- a/pipeline/frontend/metadata.go +++ b/server/pipeline/stepbuilder/metadata.go @@ -12,30 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -package frontend +package stepbuilder import ( "fmt" "net/url" "strings" - "github.com/drone/envsubst" - "go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata" "go.woodpecker-ci.org/woodpecker/v2/server/model" "go.woodpecker-ci.org/woodpecker/v2/version" ) -func EnvVarSubst(yaml string, environ map[string]string) (string, error) { - return envsubst.Eval(yaml, func(name string) string { - env := environ[name] - if strings.Contains(env, "\n") { - env = fmt.Sprintf("%q", env) - } - return env - }) -} - // MetadataFromStruct return the metadata from a pipeline will run with. func MetadataFromStruct(forge metadata.ServerForge, repo *model.Repo, pipeline, last *model.Pipeline, workflow *model.Workflow, sysURL string) metadata.Metadata { host := sysURL diff --git a/pipeline/frontend/metadata_test.go b/server/pipeline/stepbuilder/metadata_test.go similarity index 90% rename from pipeline/frontend/metadata_test.go rename to server/pipeline/stepbuilder/metadata_test.go index ed8519ac74..62e8b50d52 100644 --- a/pipeline/frontend/metadata_test.go +++ b/server/pipeline/stepbuilder/metadata_test.go @@ -12,45 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -package frontend_test +package stepbuilder import ( "testing" "github.com/stretchr/testify/assert" - "go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend" "go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata" "go.woodpecker-ci.org/woodpecker/v2/server/forge/mocks" "go.woodpecker-ci.org/woodpecker/v2/server/model" ) -func TestEnvVarSubst(t *testing.T) { - testCases := []struct { - name string - yaml string - environ map[string]string - want string - }{{ - name: "simple substitution", - yaml: `steps: - step1: - image: ${HELLO_IMAGE}`, - environ: map[string]string{"HELLO_IMAGE": "hello-world"}, - want: `steps: - step1: - image: hello-world`, - }} - - for _, testCase := range testCases { - t.Run(testCase.name, func(t *testing.T) { - result, err := frontend.EnvVarSubst(testCase.yaml, testCase.environ) - assert.NoError(t, err) - assert.EqualValues(t, testCase.want, result) - }) - } -} - func TestMetadataFromStruct(t *testing.T) { forge := mocks.NewForge(t) forge.On("Name").Return("gitea") @@ -125,7 +98,7 @@ func TestMetadataFromStruct(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - result := frontend.MetadataFromStruct(testCase.forge, testCase.repo, testCase.pipeline, testCase.last, testCase.workflow, testCase.sysURL) + result := MetadataFromStruct(testCase.forge, testCase.repo, testCase.pipeline, testCase.last, testCase.workflow, testCase.sysURL) assert.EqualValues(t, testCase.expectedMetadata, result) assert.EqualValues(t, testCase.expectedEnviron, result.Environ()) }) diff --git a/pipeline/stepBuilder.go b/server/pipeline/stepbuilder/stepBuilder.go similarity index 97% rename from pipeline/stepBuilder.go rename to server/pipeline/stepbuilder/stepBuilder.go index 84c5284753..78ceb65c03 100644 --- a/pipeline/stepBuilder.go +++ b/server/pipeline/stepbuilder/stepBuilder.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pipeline +package stepbuilder import ( "fmt" @@ -30,7 +30,6 @@ import ( yaml_types "go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/types" forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" - "go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend" "go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata" "go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml" "go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/compiler" @@ -117,7 +116,7 @@ func (b *StepBuilder) Build() (items []*Item, errorsAndWarnings error) { } func (b *StepBuilder) genItemForWorkflow(workflow *model.Workflow, axis matrix.Axis, data string) (item *Item, errorsAndWarnings error) { - workflowMetadata := frontend.MetadataFromStruct(b.Forge, b.Repo, b.Curr, b.Last, workflow, b.Host) + workflowMetadata := MetadataFromStruct(b.Forge, b.Repo, b.Curr, b.Last, workflow, b.Host) environ := b.environmentVariables(workflowMetadata, axis) // add global environment variables for substituting @@ -130,7 +129,7 @@ func (b *StepBuilder) genItemForWorkflow(workflow *model.Workflow, axis matrix.A } // substitute vars - substituted, err := frontend.EnvVarSubst(data, environ) + substituted, err := metadata.EnvVarSubst(data, environ) if err != nil { return nil, multierr.Append(errorsAndWarnings, err) } diff --git a/pipeline/stepBuilder_test.go b/server/pipeline/stepbuilder/stepBuilder_test.go similarity index 99% rename from pipeline/stepBuilder_test.go rename to server/pipeline/stepbuilder/stepBuilder_test.go index f1d0ea2ffc..559a7f6a3a 100644 --- a/pipeline/stepBuilder_test.go +++ b/server/pipeline/stepbuilder/stepBuilder_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pipeline +package stepbuilder import ( "fmt"