Skip to content

Commit

Permalink
Make sure we dont have hidden options for backend and pipeline compil…
Browse files Browse the repository at this point in the history
…er (#2123)

move options based on **os.Getenv** into flags

---------
*Sponsored by Kithara Software GmbH*
  • Loading branch information
6543 authored Aug 7, 2023
1 parent 3d47585 commit d253f8c
Show file tree
Hide file tree
Showing 28 changed files with 561 additions and 295 deletions.
12 changes: 10 additions & 2 deletions cli/exec/exec.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ import (
"github.com/woodpecker-ci/woodpecker/cli/common"
"github.com/woodpecker-ci/woodpecker/pipeline"
"github.com/woodpecker-ci/woodpecker/pipeline/backend"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/docker"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/kubernetes"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/local"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/ssh"
backendTypes "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler"
Expand All @@ -45,7 +49,7 @@ var Command = &cli.Command{
Usage: "execute a local pipeline",
ArgsUsage: "[path/to/.woodpecker.yaml]",
Action: run,
Flags: append(common.GlobalFlags, flags...),
Flags: utils.MergeSlices(common.GlobalFlags, flags, docker.Flags, ssh.Flags, kubernetes.Flags, local.Flags),
}

func run(c *cli.Context) error {
Expand Down Expand Up @@ -184,7 +188,11 @@ func execWithAxis(c *cli.Context, file, repoPath string, axis matrix.Axis) error
compiler.WithPrefix(
c.String("prefix"),
),
compiler.WithProxy(),
compiler.WithProxy(compiler.ProxyOptions{
NoProxy: c.String("backend-no-proxy"),
HTTPProxy: c.String("backend-http-proxy"),
HTTPSProxy: c.String("backend-https-proxy"),
}),
compiler.WithLocal(
c.Bool("local"),
),
Expand Down
102 changes: 19 additions & 83 deletions cli/exec/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,25 @@ var flags = []cli.Flag{
Value: "auto-detect",
},

//
// backend options for pipeline compiler
//
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_NO_PROXY", "NO_PROXY", "no_proxy"},
Usage: "if set, pass the environment variable down as \"NO_PROXY\" to steps",
Name: "backend-no-proxy",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_HTTP_PROXY", "HTTP_PROXY", "http_proxy"},
Usage: "if set, pass the environment variable down as \"NO_PROXY\" to steps",
Name: "backend-http-proxy",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_HTTPS_PROXY", "HTTPS_PROXY", "https_proxy"},
Usage: "if set, pass the environment variable down as \"HTTPS_PROXY\" to steps",
Name: "backend-https-proxy",
},

//
// Please note the below flags should match the flags from
// pipeline/frontend/metadata.go and should be kept synchronized.
Expand Down Expand Up @@ -292,87 +311,4 @@ var flags = []cli.Flag{
EnvVars: []string{"CI_FORGE_URL"},
Name: "forge-url",
},

// backend docker
&cli.BoolFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_ENABLE_IPV6"},
Name: "backend-docker-ipv6",
Usage: "backend docker enable IPV6",
Value: false,
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_NETWORK"},
Name: "backend-docker-network",
Usage: "backend docker network",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_VOLUMES"},
Name: "backend-docker-volumes",
Usage: "backend docker volumes (comma separated)",
},

// backend ssh
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_ADDRESS"},
Name: "backend-ssh-address",
Usage: "backend ssh address",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_USER"},
Name: "backend-ssh-user",
Usage: "backend ssh user",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY"},
Name: "backend-ssh-key",
Usage: "backend ssh key file",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY_PASSWORD"},
Name: "backend-ssh-key-password",
Usage: "backend ssh key password",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_PASSWORD"},
Name: "backend-ssh-password",
Usage: "backend ssh password",
},

// backend k8s
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_NAMESPACE"},
Name: "backend-k8s-namespace",
Usage: "backend k8s namespace",
Value: "woodpecker",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_VOLUME_SIZE"},
Name: "backend-k8s-volume-size",
Usage: "backend k8s volume size (default 10G)",
Value: "10G",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_CLASS"},
Name: "backend-k8s-storage-class",
Usage: "backend k8s storage class",
Value: "",
},
&cli.BoolFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_RWX"},
Name: "backend-k8s-storage-rwx",
Usage: "backend k8s storage access mode, should ReadWriteMany (RWX) instead of ReadWriteOnce (RWO) be used? (default: true)",
Value: true,
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_LABELS"},
Name: "backend-k8s-pod-labels",
Usage: "backend k8s additional worker pod labels",
Value: "",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS"},
Name: "backend-k8s-pod-annotations",
Usage: "backend k8s additional worker pod annotations",
Value: "",
},
}
101 changes: 2 additions & 99 deletions cmd/agent/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,9 @@ import (
"time"

"github.com/urfave/cli/v2"

"github.com/woodpecker-ci/woodpecker/cmd/common"
)

var flags = append([]cli.Flag{
var flags = []cli.Flag{
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_SERVER"},
Name: "server",
Expand Down Expand Up @@ -99,99 +97,4 @@ var flags = append([]cli.Flag{
Usage: "backend engine to run pipelines on",
Value: "auto-detect",
},

// backend docker
&cli.BoolFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_ENABLE_IPV6"},
Name: "backend-docker-ipv6",
Usage: "backend docker enable IPV6",
Value: false,
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_NETWORK"},
Name: "backend-docker-network",
Usage: "backend docker network",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_VOLUMES"},
Name: "backend-docker-volumes",
Usage: "backend docker volumes (comma separated)",
},

// backend ssh
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_ADDRESS"},
Name: "backend-ssh-address",
Usage: "backend ssh address",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_USER"},
Name: "backend-ssh-user",
Usage: "backend ssh user",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY"},
Name: "backend-ssh-key",
Usage: "backend ssh key file",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY_PASSWORD"},
Name: "backend-ssh-key-password",
Usage: "backend ssh key password",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_SSH_PASSWORD"},
Name: "backend-ssh-password",
Usage: "backend ssh password",
},

// backend k8s
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_NAMESPACE"},
Name: "backend-k8s-namespace",
Usage: "backend k8s namespace",
Value: "woodpecker",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_VOLUME_SIZE"},
Name: "backend-k8s-volume-size",
Usage: "backend k8s volume size (default 10G)",
Value: "10G",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_CLASS"},
Name: "backend-k8s-storage-class",
Usage: "backend k8s storage class",
Value: "",
},
&cli.BoolFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_RWX"},
Name: "backend-k8s-storage-rwx",
Usage: "backend k8s storage access mode, should ReadWriteMany (RWX) instead of ReadWriteOnce (RWO) be used? (default: true)",
Value: true,
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_LABELS"},
Name: "backend-k8s-pod-labels",
Usage: "backend k8s additional worker pod labels",
Value: "",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS"},
Name: "backend-k8s-pod-annotations",
Usage: "backend k8s additional worker pod annotations",
Value: "",
},
&cli.IntFlag{
EnvVars: []string{"WOODPECKER_CONNECT_RETRY_COUNT"},
Name: "connect-retry-count",
Usage: "number of times to retry connecting to the server",
Value: 5,
},
&cli.DurationFlag{
EnvVars: []string{"WOODPECKER_CONNECT_RETRY_DELAY"},
Name: "connect-retry-delay",
Usage: "duration to wait before retrying to connect to the server",
Value: time.Second * 2,
},
}, common.GlobalLoggerFlags...)
}
8 changes: 7 additions & 1 deletion cmd/agent/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,12 @@ import (
_ "github.com/joho/godotenv/autoload"
"github.com/urfave/cli/v2"

"github.com/woodpecker-ci/woodpecker/cmd/common"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/docker"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/kubernetes"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/local"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/ssh"
"github.com/woodpecker-ci/woodpecker/shared/utils"
"github.com/woodpecker-ci/woodpecker/version"
)

Expand All @@ -37,7 +43,7 @@ func main() {
Action: pinger,
},
}
app.Flags = flags
app.Flags = utils.MergeSlices(flags, common.GlobalLoggerFlags, docker.Flags, ssh.Flags, kubernetes.Flags, local.Flags)

if err := app.Run(os.Args); err != nil {
fmt.Fprintln(os.Stderr, err)
Expand Down
18 changes: 18 additions & 0 deletions cmd/server/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,24 @@ var flags = append([]cli.Flag{
Value: true,
},
//
// backend options for pipeline compiler
//
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_NO_PROXY", "NO_PROXY", "no_proxy"},
Usage: "if set, pass the environment variable down as \"NO_PROXY\" to steps",
Name: "backend-no-proxy",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_HTTP_PROXY", "HTTP_PROXY", "http_proxy"},
Usage: "if set, pass the environment variable down as \"NO_PROXY\" to steps",
Name: "backend-http-proxy",
},
&cli.StringFlag{
EnvVars: []string{"WOODPECKER_BACKEND_HTTPS_PROXY", "HTTPS_PROXY", "https_proxy"},
Usage: "if set, pass the environment variable down as \"HTTPS_PROXY\" to steps",
Name: "backend-https-proxy",
},
//
// resource limit parameters
//
&cli.DurationFlag{
Expand Down
5 changes: 5 additions & 0 deletions cmd/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,11 @@ func setupEvilGlobals(c *cli.Context, v store.Store, f forge.Forge) {
server.Config.Pipeline.Limits.CPUShares = c.Int64("limit-cpu-shares")
server.Config.Pipeline.Limits.CPUSet = c.String("limit-cpu-set")

// backend options for pipeline compiler
server.Config.Pipeline.Proxy.No = c.String("backend-no-proxy")
server.Config.Pipeline.Proxy.HTTP = c.String("backend-http-proxy")
server.Config.Pipeline.Proxy.HTTPS = c.String("backend-https-proxy")

// server configuration
server.Config.Server.Cert = c.String("server-cert")
server.Config.Server.Key = c.String("server-key")
Expand Down
51 changes: 45 additions & 6 deletions pipeline/backend/docker/docker.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,16 @@ package docker
import (
"context"
"io"
"net/http"
"os"
"path/filepath"
"runtime"
"strings"

"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/api/types/volume"
"github.com/docker/go-connections/tlsconfig"
"github.com/moby/moby/client"
"github.com/moby/moby/pkg/jsonmessage"
"github.com/moby/moby/pkg/stdcopy"
Expand Down Expand Up @@ -67,20 +70,56 @@ func (e *docker) IsAvailable(context.Context) bool {
return err == nil
}

// Load new client for Docker Engine using environment variables.
func (e *docker) Load(ctx context.Context) error {
cl, err := client.NewClientWithOpts(client.FromEnv)
func httpClientOfOpts(dockerCertPath string, verifyTLS bool) *http.Client {
if dockerCertPath == "" {
return nil
}

options := tlsconfig.Options{
CAFile: filepath.Join(dockerCertPath, "ca.pem"),
CertFile: filepath.Join(dockerCertPath, "cert.pem"),
KeyFile: filepath.Join(dockerCertPath, "key.pem"),
InsecureSkipVerify: !verifyTLS,
}
tlsConf, err := tlsconfig.Client(options)
if err != nil {
return err
log.Error().Err(err).Msg("could not create http client out of docker backend options")
return nil
}
e.client = cl

return &http.Client{
Transport: &http.Transport{TLSClientConfig: tlsConf},
CheckRedirect: client.CheckRedirect,
}
}

// Load new client for Docker Engine using environment variables.
func (e *docker) Load(ctx context.Context) error {
c, ok := ctx.Value(backend.CliContext).(*cli.Context)
if !ok {
return backend.ErrNoCliContextFound
}
e.enableIPv6 = c.Bool("backend-docker-ipv6")

var dockerClientOpts []client.Opt
if httpClient := httpClientOfOpts(c.String("backend-docker-cert"), c.Bool("backend-docker-tls-verify")); httpClient != nil {
dockerClientOpts = append(dockerClientOpts, client.WithHTTPClient(httpClient))
}
if dockerHost := c.String("backend-docker-host"); dockerHost != "" {
dockerClientOpts = append(dockerClientOpts, client.WithHost(dockerHost))
}
if dockerAPIVersion := c.String("backend-docker-api-version"); dockerAPIVersion != "" {
dockerClientOpts = append(dockerClientOpts, client.WithVersion(dockerAPIVersion))
} else {
dockerClientOpts = append(dockerClientOpts, client.WithAPIVersionNegotiation())
}

cl, err := client.NewClientWithOpts(dockerClientOpts...)
if err != nil {
return err
}
e.client = cl

e.enableIPv6 = c.Bool("backend-docker-ipv6")
e.network = c.String("backend-docker-network")

volumes := strings.Split(c.String("backend-docker-volumes"), ",")
Expand Down
Loading

0 comments on commit d253f8c

Please sign in to comment.