diff --git a/cmd/sonobuoy/app/args.go b/cmd/sonobuoy/app/args.go index 5aac1fbf2..0eecaa1f7 100644 --- a/cmd/sonobuoy/app/args.go +++ b/cmd/sonobuoy/app/args.go @@ -50,6 +50,7 @@ const ( e2eSkipFlag = "e2e-skip" e2eParallelFlag = "e2e-parallel" e2eRegistryConfigFlag = "e2e-repo-config" + e2eDockerConfigFileFlag = "e2e-docker-config-file" e2eRegistryFlag = "e2e-repo" pluginImageFlag = "plugin-image" filenameFlag = "filename" @@ -224,7 +225,7 @@ func AddSonobuoyConfigFlag(cfg *SonobuoyConfig, flags *pflag.FlagSet) { // AddLegacyE2EFlags is a way to add flags which target the e2e plugin specifically // by leveraging the existing flags. They typically wrap other fields (like the env var // overrides) and modify those. -func AddLegacyE2EFlags(env *PluginEnvVars, pluginTransforms *map[string][]func(*manifest.Manifest) error, fs *pflag.FlagSet) { +func AddLegacyE2EFlags(cfg *SonobuoyConfig, env *PluginEnvVars, pluginTransforms *map[string][]func(*manifest.Manifest) error, fs *pflag.FlagSet) { m := &Mode{ env: env, name: "", @@ -289,6 +290,15 @@ func AddLegacyE2EFlags(env *PluginEnvVars, pluginTransforms *map[string][]func(* &envVarModierFlag{plugin: e2ePlugin, field: "KUBE_TEST_REPO", PluginEnvVars: *env}, e2eRegistryFlag, "Specify a registry to use as the default for pulling Kubernetes test images. Same as providing --e2e-repo-config but specifying the same repo repeatedly.", ) + + fs.Var( + &e2eDockerConfigFlag{ + plugin: e2ePlugin, + config: cfg, + transforms: *pluginTransforms, + }, e2eDockerConfigFileFlag, + "A docker credentials configuration file used which contains authorization token that can be used to pull images from certain private registries provided by the users", + ) } // AddRBACModeFlags adds an E2E Argument with the provided default. @@ -563,9 +573,11 @@ func (f *e2eRepoFlag) Set(str string) error { } f.transforms[f.plugin] = append(f.transforms[f.plugin], func(m *manifest.Manifest) error { - m.ConfigMap = map[string]string{ - name: string(fData), + if m.ConfigMap == nil { + m.ConfigMap = map[string]string{} } + m.ConfigMap[name] = string(fData) + m.Spec.Env = append(m.Spec.Env, corev1.EnvVar{ Name: "KUBE_TEST_REPO_LIST", Value: fmt.Sprintf("/tmp/sonobuoy/config/%v", name), @@ -575,6 +587,40 @@ func (f *e2eRepoFlag) Set(str string) error { return nil } +type e2eDockerConfigFlag struct { + plugin string + filename string + config *SonobuoyConfig + + transforms map[string][]func(*manifest.Manifest) error + // Value to put in the configmap as the filename. Defaults to filename. + filenameOverride string +} + +func (f *e2eDockerConfigFlag) String() string { return f.filename } +func (f *e2eDockerConfigFlag) Type() string { return "json-filepath" } +func (f *e2eDockerConfigFlag) Set(str string) error { + f.config.E2EDockerConfigFile = str + name := filepath.Base(str) + if len(f.filenameOverride) > 0 { + name = f.filenameOverride + } + fData, err := os.ReadFile(str) + if err != nil { + return errors.Wrapf(err, "failed to read file %q", str) + } + + f.transforms[e2ePlugin] = append(f.transforms[e2ePlugin], func(m *manifest.Manifest) error { + if m.ConfigMap == nil { + m.ConfigMap = map[string]string{} + } + m.ConfigMap[name] = string(fData) + return nil + + }) + return nil +} + // The ssh-key flag needs to store the path to the ssh key but also // wire up the e2e plugin for using it. type sshPathFlag struct { diff --git a/cmd/sonobuoy/app/gen.go b/cmd/sonobuoy/app/gen.go index aca4ecc99..73feac349 100644 --- a/cmd/sonobuoy/app/gen.go +++ b/cmd/sonobuoy/app/gen.go @@ -21,6 +21,8 @@ package app import ( "fmt" "os" + "strconv" + "strings" "time" "github.com/vmware-tanzu/sonobuoy/pkg/client" @@ -94,7 +96,7 @@ func GenFlagSet(cfg *genFlags, rbac RBACMode) *pflag.FlagSet { AddPluginSetFlag(&cfg.plugins, genset) AddPluginEnvFlag(&cfg.pluginEnvs, genset) - AddLegacyE2EFlags(&cfg.pluginEnvs, &cfg.pluginTransforms, genset) + AddLegacyE2EFlags(&cfg.sonobuoyConfig, &cfg.pluginEnvs, &cfg.pluginTransforms, genset) AddNodeSelectorsFlag(&cfg.nodeSelectors, genset) @@ -165,6 +167,15 @@ func (g *genFlags) Config() (*client.GenConfig, error) { k8sVersion = g.k8sVersion.String() } + if g.sonobuoyConfig.E2EDockerConfigFile != "" { + if err := verifyKubernetesVersion(k8sVersion); err != nil { + return nil, err + } + if g.sonobuoyConfig.ImagePullSecrets == "" { + g.sonobuoyConfig.ImagePullSecrets = "auth-repo-cred" + } + } + return &client.GenConfig{ Config: &g.sonobuoyConfig.Config, EnableRBAC: rbacEnabled, @@ -207,6 +218,7 @@ func NewCmdGen() *cobra.Command { Args: cobra.ExactArgs(0), } GenCommand.Flags().AddFlagSet(GenFlagSet(&genflags, EnabledRBACMode)) + return GenCommand } @@ -261,3 +273,25 @@ func getClient(kubeconfig *Kubeconfig) (*kubernetes.Clientset, error) { return client, kubeError } + +func verifyKubernetesVersion(k8sVersion string) error { + parts := versionMatchRE.FindStringSubmatch(k8sVersion) + if parts == nil { + return fmt.Errorf("could not parse %q as version", k8sVersion) + } + numbers, _ := parts[1], parts[2] + + versionInfo := strings.Split(numbers, ".") + minorVersion := versionInfo[1] + + minorVersionInt, err := strconv.ParseInt(minorVersion, 10, 0) + if err != nil { + return err + } + + if minorVersionInt < 27 { + err = fmt.Errorf("e2e-docker-config-file is only supported for Kubernetes 1.27 or later") + } + + return err +} diff --git a/cmd/sonobuoy/app/gen_test.go b/cmd/sonobuoy/app/gen_test.go new file mode 100644 index 000000000..423a64a7c --- /dev/null +++ b/cmd/sonobuoy/app/gen_test.go @@ -0,0 +1,53 @@ +/* +Copyright the Sonobuoy contributors 2019 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "testing" +) + +func TestVerifyKubernetesVersion(t *testing.T) { + testCases := []struct { + desc string + k8sVersion string + expectErr string + }{ + { + desc: "Usage of e2e-docker-config-file flag with Kubernetes versions below 1.27 should throw error", + k8sVersion: "1.26-alpha-3", + expectErr: "e2e-docker-config-file is only supported for Kubernetes 1.27 or later", + }, + { + desc: "Usage of e2e-docker-config-file flag along with Kubernetes versions 1.27 or later should work as expected", + k8sVersion: "1.27", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.desc, func(t *testing.T) { + err := verifyKubernetesVersion(testCase.k8sVersion) + if err != nil { + if len(testCase.expectErr) == 0 { + t.Fatalf("Expected nil error but got %v", err) + } + if err.Error() != testCase.expectErr { + t.Fatalf("Expected error %q but got %q", err, testCase.expectErr) + } + } + }) + + } +} diff --git a/cmd/sonobuoy/app/images.go b/cmd/sonobuoy/app/images.go index 7bd97df78..78769437f 100644 --- a/cmd/sonobuoy/app/images.go +++ b/cmd/sonobuoy/app/images.go @@ -451,7 +451,6 @@ func translateRegistry(imageURL string, customRegistry string, customRegistryLis if len(customRegistry) > 0 { return fmt.Sprintf("%s/%s", customRegistry, parts[countParts-1]) } - // For now, if not given a customRegistry, assume they gave the customRegistryList as non-nil. switch registryAndUser { case "gcr.io/e2e-test-images": diff --git a/cmd/sonobuoy/app/root.go b/cmd/sonobuoy/app/root.go index 28df6e0d9..ea4feed97 100644 --- a/cmd/sonobuoy/app/root.go +++ b/cmd/sonobuoy/app/root.go @@ -20,6 +20,7 @@ import ( "errors" "flag" "fmt" + "regexp" "sync" "github.com/sirupsen/logrus" @@ -34,6 +35,10 @@ import ( // and cause a panic otherwise. var once sync.Once +// versionMatchRE splits a version string into numeric and "extra" parts +// source: https://github.com/kubernetes/kubernetes/blob/af1bf4306709020ed2002618e099b3d0acf3c7b5/staging/src/k8s.io/apimachinery/pkg/util/version/version.go#L37 +var versionMatchRE = regexp.MustCompile(`^\s*v?([0-9]+(?:\.[0-9]+)*)(.*)*$`) + func NewSonobuoyCommand() *cobra.Command { cmds := &cobra.Command{ Use: "sonobuoy", @@ -106,8 +111,10 @@ func prerunChecks(cmd *cobra.Command, args []string) error { // Getting a list of all flags provided by the user. flagsSet := map[string]bool{} flagsDebug := []string{} + flagArgs := map[string]string{} cmd.Flags().Visit(func(f *pflag.Flag) { flagsSet[f.Name] = true + flagArgs[f.Name] = f.Value.String() flagsDebug = append(flagsDebug, fmt.Sprintf("%v=%v", f.Name, f.Value.String())) }) @@ -135,6 +142,13 @@ func prerunChecks(cmd *cobra.Command, args []string) error { return fmt.Errorf("%v and %v flags are both set and may collide", e2eRegistryConfigFlag, e2eRegistryFlag) } + if flagsSet[e2eDockerConfigFileFlag] && flagsSet["kubernetes-version"] { + k8sVersion := flagArgs["kubernetes-version"] + if err := verifyKubernetesVersion(k8sVersion); err != nil { + return err + } + } + return nil } diff --git a/pkg/client/gen.go b/pkg/client/gen.go index fba885cfd..2367080ff 100644 --- a/pkg/client/gen.go +++ b/pkg/client/gen.go @@ -25,6 +25,7 @@ import ( "fmt" "io" "os" + "path/filepath" "sort" "strings" @@ -51,7 +52,8 @@ const ( aggregatorEnvOverrideKey = `sonobuoy` - envVarKeyExtraArgs = "E2E_EXTRA_ARGS" + envVarKeyExtraArgs = "E2E_EXTRA_ARGS" + defaultImagePullSecretName = "auth-repo-cred" // sonobuoyKey is just a true/false env to indicate that the container was launched/tagged by Sonobuoy. sonobuoyKey = "SONOBUOY" @@ -164,6 +166,7 @@ func (*SonobuoyClient) GenerateManifestAndPlugins(cfg *GenConfig) ([]byte, []*ma if len(p.ConfigMap) == 0 { continue } + configs[p.SonobuoyConfig.PluginName] = p.ConfigMap p.ExtraVolumes = append(p.ExtraVolumes, manifest.Volume{ @@ -218,6 +221,11 @@ func generateYAMLComponents(w io.Writer, cfg *GenConfig, plugins []*manifest.Man if err := generateServiceAcct(w, cfg); err != nil { return err } + if cfg.Config.E2EDockerConfigFile != "" { + if err := generateRegistrySecret(w, cfg); err != nil { + return err + } + } if err := generateRBAC(w, cfg); err != nil { return err } @@ -227,6 +235,7 @@ func generateYAMLComponents(w io.Writer, cfg *GenConfig, plugins []*manifest.Man if err := generateSecret(w, cfg); err != nil { return err } + if err := generatePluginConfigmap(w, cfg, plugins); err != nil { return err } @@ -261,7 +270,9 @@ func generateAdditionalConfigmaps(w io.Writer, cfg *GenConfig, configs map[strin sort.Strings(filenames) for _, filename := range filenames { cm.Data[filename] = configs[pluginName][filename] + } + if err := appendAsYAML(w, cm); err != nil { return err } @@ -282,6 +293,7 @@ func generatePluginConfigmap(w io.Writer, cfg *GenConfig, plugins []*manifest.Ma } cm.Data[fmt.Sprintf("plugin-%v.yaml", i)] = strings.TrimSpace(string(b)) } + return appendAsYAML(w, cm) } @@ -309,6 +321,24 @@ func appendAsYAML(w io.Writer, o kuberuntime.Object) error { return err } +func generateRegistrySecret(w io.Writer, cfg *GenConfig) error { + contents, err := os.ReadFile(cfg.Config.E2EDockerConfigFile) + if err != nil { + return fmt.Errorf("error reading docker config file: %v", err) + } + s := &corev1.Secret{ + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{corev1.DockerConfigJsonKey: contents}, + } + s.Name = cfg.Config.ImagePullSecrets + s.Namespace = cfg.Config.Namespace + + s.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Secret"}) + + return appendAsYAML(w, s) + +} + func generateSecret(w io.Writer, cfg *GenConfig) error { if len(cfg.SSHKeyPath) == 0 { return nil @@ -790,14 +820,14 @@ func E2EManifest(cfg *GenConfig) *manifest.Manifest { } m.PodSpec.PodSpec.NodeSelector = map[string]string{"kubernetes.io/os": "linux"} - m.Spec.Env = updateExtraArgs(m.Spec.Env, cfg.Config.ProgressUpdatesPort) + m.Spec.Env = updateExtraArgs(m.Spec.Env, cfg.Config.ProgressUpdatesPort, cfg.Config.E2EDockerConfigFile) return m } // updateExtraArgs adds the flag expected by the e2e plugin for the progress report URL. // If no port is given, the default "8099" is used. -func updateExtraArgs(envs []corev1.EnvVar, port string) []corev1.EnvVar { +func updateExtraArgs(envs []corev1.EnvVar, port, e2eDockerConfigFile string) []corev1.EnvVar { for _, env := range envs { // If set by user, just leave as-is. if env.Name == envVarKeyExtraArgs { @@ -808,6 +838,11 @@ func updateExtraArgs(envs []corev1.EnvVar, port string) []corev1.EnvVar { port = config.DefaultProgressUpdatesPort } val := fmt.Sprintf("--progress-report-url=http://localhost:%v/progress", port) + if e2eDockerConfigFile != "" { + credFile := filepath.Base(e2eDockerConfigFile) + registryCredLocation := fmt.Sprintf("%s/%s", sonobuoyDefaultConfigDir, credFile) + val += fmt.Sprintf(" --e2e-docker-config-file=%s", registryCredLocation) + } envs = append(envs, corev1.EnvVar{Name: envVarKeyExtraArgs, Value: val}) return envs } diff --git a/pkg/client/interfaces.go b/pkg/client/interfaces.go index 10fca773d..d493fdb98 100644 --- a/pkg/client/interfaces.go +++ b/pkg/client/interfaces.go @@ -110,7 +110,7 @@ func (gc *GenConfig) Validate() error { } for key, value := range m.ConfigMap { - if strings.HasSuffix(key, ".yml") || strings.HasSuffix(key, ".yaml") { + if strings.HasSuffix(key, ".yml") || strings.HasSuffix(key, ".yaml") || strings.HasSuffix(key, ".json") { var i interface{} if err := yaml.Unmarshal([]byte(value), &i); err != nil { return fmt.Errorf("failed to parse value of key %v in ConfigMap for plugin %v: %v", key, m.SonobuoyConfig.PluginName, err) diff --git a/pkg/config/config.go b/pkg/config/config.go index a90abc9fd..9861dd852 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -155,6 +155,7 @@ type Config struct { AggregatorPermissions string `json:"AggregatorPermissions" mapstructure:"AggregatorPermissions"` ServiceAccountName string `json:"ServiceAccountName" mapstructure:"ServiceAccountName"` ExistingServiceAccount bool `json:"ExistingServiceAccount,omitempty" mapstructure:"ExistingServiceAccount,omitempty"` + E2EDockerConfigFile string `json:"E2EDockerConfigFile,omitempty" mapstructure:"E2EDockerConfigFile,omitempty"` NamespacePSAEnforceLevel string `json:"NamespacePSAEnforceLevel,omitempty" mapstructure:"NamespacePSAEnforceLevel,omitempty"` // ProgressUpdatesPort is the port on which the Sonobuoy worker will listen for status updates from its plugin. diff --git a/pkg/image/imageversion.go b/pkg/image/imageversion.go index 4267fd9b3..645a91f0b 100644 --- a/pkg/image/imageversion.go +++ b/pkg/image/imageversion.go @@ -18,11 +18,12 @@ package image import ( "fmt" - "github.com/sirupsen/logrus" "io" "net/http" "strings" + "github.com/sirupsen/logrus" + "github.com/hashicorp/go-version" "github.com/pkg/errors" "github.com/vmware-tanzu/sonobuoy/pkg/config"