From 93bc2edd3e2b086d72427afa6170059af95f8c10 Mon Sep 17 00:00:00 2001 From: Shirley Xiaolin Xu Date: Mon, 27 Sep 2021 15:31:40 -0400 Subject: [PATCH 01/13] Allowing the declaration of sidecars in the k8s plugin Also some other minor refactoring --- builtin/k8s/platform.go | 600 ++++++++++++++++++++++++---------------- 1 file changed, 360 insertions(+), 240 deletions(-) diff --git a/builtin/k8s/platform.go b/builtin/k8s/platform.go index 9b9c875c8ad..392cefd9e44 100644 --- a/builtin/k8s/platform.go +++ b/builtin/k8s/platform.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "strconv" "strings" "time" @@ -290,318 +289,392 @@ func (p *Platform) resourceDeploymentStatus( return nil } -// resourceDeploymentCreate creates the Kubernetes deployment. -func (p *Platform) resourceDeploymentCreate( - ctx context.Context, +func configureK8sContainer( + name string, + image string, + ports []*Port, + envVars map[string]string, + probe *Probe, + probePath string, + cpu *ResourceConfig, + memory *ResourceConfig, + resources map[string]string, + command *[]string, + args *[]string, + scratchSpace []string, + volumes []corev1.Volume, log hclog.Logger, - src *component.Source, - img *docker.Image, - deployConfig *component.DeploymentConfig, - ui terminal.UI, - - result *Deployment, - state *Resource_Deployment, - csinfo *clientsetInfo, - sg terminal.StepGroup, -) error { - // Prepare our namespace and override if set. - ns := csinfo.Namespace - if p.config.Namespace != "" { - ns = p.config.Namespace +) (*corev1.Container, error) { + // If the user is using the latest tag, then don't specify an overriding pull policy. + // This by default means kubernetes will always pull so that latest is useful. + splitImage := strings.Split(image, ":") + var pullPolicy corev1.PullPolicy + if len(splitImage) == 1 { + // If no tag is set, docker will default to "latest", and we always want to pull. + pullPolicy = corev1.PullAlways + } else if len(splitImage) == 2 { + tag := splitImage[1] + if tag == "latest" { + // Always pull a latest image, so the k8s workers don't use their local cache and ignore new changes. + pullPolicy = corev1.PullAlways + } else { + // A tag is present, we can use k8s worker caching + pullPolicy = corev1.PullIfNotPresent + } + } else { + return nil, status.Errorf(codes.InvalidArgument, "image %s is in an invalid format", image) } - step := sg.Add("") - defer func() { step.Abort() }() - step.Update("Kubernetes client connected to %s with namespace %s", csinfo.Config.Host, ns) - step.Done() - - step = sg.Add("Preparing deployment...") - - clientSet := csinfo.Clientset - deployClient := clientSet.AppsV1().Deployments(ns) - - // Determine if we have a deployment that we manage already - create := false - deployment, err := deployClient.Get(ctx, result.Name, metav1.GetOptions{}) - if errors.IsNotFound(err) { - deployment = result.newDeployment(result.Name) - create = true - err = nil - } - if err != nil { - return err + var k8sPorts []corev1.ContainerPort + for _, port := range ports { + k8sPorts = append(k8sPorts, corev1.ContainerPort{ + Name: port.Name, + ContainerPort: int32(port.Port), + HostPort: int32(port.HostPort), + HostIP: port.HostIP, + Protocol: corev1.Protocol(strings.TrimSpace(strings.ToUpper(port.Protocol))), + }) } - // Setup our port configuration - if p.config.ServicePort == 0 && p.config.Ports == nil { - // nothing defined, set up the defaults - p.config.Ports = make([]map[string]string, 1) - p.config.Ports[0] = map[string]string{"port": strconv.Itoa(DefaultServicePort), "name": "http"} - } else if p.config.ServicePort > 0 && p.config.Ports == nil { - // old ServicePort var is used, so set it up in our Ports map to be used - p.config.Ports = make([]map[string]string, 1) - p.config.Ports[0] = map[string]string{"port": strconv.Itoa(int(p.config.ServicePort)), "name": "http"} - } else if p.config.ServicePort > 0 && len(p.config.Ports) > 0 { - // both defined, this is an error - return fmt.Errorf("Cannot define both 'service_port' and 'ports'. Use" + - " 'ports' for configuring multiple container ports.") + if envVars == nil { + envVars = make(map[string]string) } - // Build our env vars - env := []corev1.EnvVar{ - { - Name: "PORT", - Value: fmt.Sprint(p.config.Ports[0]["port"]), - }, - } - for k, v := range p.config.StaticEnvVars { - env = append(env, corev1.EnvVar{ - Name: k, - Value: v, - }) - } - for k, v := range deployConfig.Env() { - env = append(env, corev1.EnvVar{ - Name: k, - Value: v, - }) + // assume the first port defined is the 'main' port to use + var defaultPort int + if len(k8sPorts) != 0 { + defaultPort = int(k8sPorts[0].ContainerPort) + envVars["PORT"] = fmt.Sprintf("%d", defaultPort) } - - // If no count is specified, presume that the user is managing the replica - // count some other way (perhaps manual scaling, perhaps a pod autoscaler). - // Either way if they don't specify a count, we should be sure we don't send one. - if p.config.Count > 0 { - deployment.Spec.Replicas = &p.config.Count + var k8sEnvVars []corev1.EnvVar + for k, v := range envVars { + k8sEnvVars = append(k8sEnvVars, corev1.EnvVar{Name: k, Value: v}) } - // Set our ID on the label. We use this ID so that we can have a key - // to route to multiple versions during release management. - deployment.Spec.Template.Labels[labelId] = result.Id - - // Version label duplicates "labelId" to support services like Istio that - // expect pods to be labled with 'version' - deployment.Spec.Template.Labels["version"] = result.Id - - // Apply user defined labels - for k, v := range p.config.Labels { - deployment.Spec.Template.Labels[k] = v - } + initialDelaySeconds := int32(5) + timeoutSeconds := int32(5) + failureThreshold := int32(5) - // If the user is using the latest tag, then don't specify an overriding pull policy. - // This by default means kubernetes will always pull so that latest is useful. - pullPolicy := corev1.PullIfNotPresent - if img.Tag == "latest" { - pullPolicy = "" + if probe != nil { + if probe.InitialDelaySeconds != 0 { + initialDelaySeconds = int32(probe.InitialDelaySeconds) + } + if probe.TimeoutSeconds != 0 { + timeoutSeconds = int32(probe.TimeoutSeconds) + } + if probe.FailureThreshold != 0 { + failureThreshold = int32(probe.FailureThreshold) + } } // Get container resource limits and requests var resourceLimits = make(map[corev1.ResourceName]k8sresource.Quantity) var resourceRequests = make(map[corev1.ResourceName]k8sresource.Quantity) - if p.config.CPU != nil { - if p.config.CPU.Requested != "" { - q, err := k8sresource.ParseQuantity(p.config.CPU.Requested) + if cpu != nil { + if cpu.Requested != "" { + q, err := k8sresource.ParseQuantity(cpu.Requested) if err != nil { - return err + return nil, + status.Errorf(codes.InvalidArgument, "failed to parse cpu request %s to k8s quantity: %s", cpu.Requested, err) } - resourceRequests[corev1.ResourceCPU] = q } - if p.config.CPU.Limit != "" { - q, err := k8sresource.ParseQuantity(p.config.CPU.Limit) + if cpu.Limit != "" { + q, err := k8sresource.ParseQuantity(cpu.Limit) if err != nil { - return err + return nil, + status.Errorf(codes.InvalidArgument, "failed to parse cpu limit %s to k8s quantity: %s", cpu.Limit, err) } - resourceLimits[corev1.ResourceCPU] = q } } - if p.config.Memory != nil { - if p.config.Memory.Requested != "" { - q, err := k8sresource.ParseQuantity(p.config.Memory.Requested) + if memory != nil { + if memory.Requested != "" { + q, err := k8sresource.ParseQuantity(memory.Requested) if err != nil { - return err + return nil, + status.Errorf(codes.InvalidArgument, "failed to parse memory requested %s to k8s quantity: %s", memory.Requested, err) } - resourceRequests[corev1.ResourceMemory] = q } - if p.config.Memory.Limit != "" { - q, err := k8sresource.ParseQuantity(p.config.Memory.Limit) + if memory.Limit != "" { + q, err := k8sresource.ParseQuantity(memory.Limit) if err != nil { - return err + return nil, + status.Errorf(codes.InvalidArgument, "failed to parse memory limit %s to k8s quantity: %s", memory.Limit, err) } - resourceLimits[corev1.ResourceMemory] = q } } - for k, v := range p.config.Resources { - if strings.HasPrefix(k, "limits_") { - limitKey := strings.Split(k, "_") - resourceName := corev1.ResourceName(limitKey[1]) + for k, v := range resources { + if strings.HasPrefix(k, "limits_") || strings.HasPrefix(k, "requests_") { + key := strings.Split(k, "_") + resourceName := corev1.ResourceName(key[1]) quantity, err := k8sresource.ParseQuantity(v) if err != nil { - return err + return nil, + status.Errorf(codes.InvalidArgument, "failed to parse resource %s to k8s quantity: %s", v, err) } resourceLimits[resourceName] = quantity - } else if strings.HasPrefix(k, "requests_") { - reqKey := strings.Split(k, "_") - resourceName := corev1.ResourceName(reqKey[1]) - - quantity, err := k8sresource.ParseQuantity(v) - if err != nil { - return err - } - resourceRequests[resourceName] = quantity } else { log.Warn("ignoring unrecognized k8s resources key: %q", k) } } - _, cpuLimit := resourceLimits[corev1.ResourceCPU] - _, cpuRequest := resourceRequests[corev1.ResourceCPU] - - if p.config.AutoscaleConfig != nil && !(cpuLimit || cpuRequest) { - ui.Output("For autoscaling in Kubernetes to work, a deployment must specify "+ - "cpu resource limits and requests. Otherwise the metrics-server will not properly be able "+ - "to scale your deployment.", terminal.WithWarningStyle()) - } - resourceRequirements := corev1.ResourceRequirements{ Limits: resourceLimits, Requests: resourceRequests, } - containerPorts := make([]corev1.ContainerPort, len(p.config.Ports)) - for i, cp := range p.config.Ports { - hostPort, _ := strconv.ParseInt(cp["host_port"], 10, 32) - port, _ := strconv.ParseInt(cp["port"], 10, 32) - - containerPorts[i] = corev1.ContainerPort{ - Name: cp["name"], - ContainerPort: int32(port), - HostPort: int32(hostPort), - HostIP: cp["host_ip"], - Protocol: corev1.ProtocolTCP, - } + var volumeMounts []corev1.VolumeMount + for idx, scratchSpaceLocation := range scratchSpace { + volumeMounts = append( + volumeMounts, + corev1.VolumeMount{ + // We know all the volumes are identical + Name: volumes[idx].Name, + MountPath: scratchSpaceLocation, + }, + ) } - // assume the first port defined is the 'main' port to use - defaultPort := int(containerPorts[0].ContainerPort) + container := corev1.Container{ + Name: name, + Image: image, + ImagePullPolicy: pullPolicy, + Env: k8sEnvVars, + Resources: resourceRequirements, + VolumeMounts: volumeMounts, + } - initialDelaySeconds := int32(5) - timeoutSeconds := int32(5) - failureThreshold := int32(5) - if p.config.Probe != nil { - if p.config.Probe.InitialDelaySeconds != 0 { - initialDelaySeconds = int32(p.config.Probe.InitialDelaySeconds) - } - if p.config.Probe.TimeoutSeconds != 0 { - timeoutSeconds = int32(p.config.Probe.TimeoutSeconds) - } - if p.config.Probe.FailureThreshold != 0 { - failureThreshold = int32(p.config.Probe.FailureThreshold) - } + if len(k8sPorts) > 0 { + container.Ports = k8sPorts + } + if command != nil { + container.Command = *command + } + if args != nil { + container.Args = *args } - container := corev1.Container{ - Name: result.Name, - Image: img.Name(), - ImagePullPolicy: pullPolicy, - Ports: containerPorts, - LivenessProbe: &corev1.Probe{ - Handler: corev1.Handler{ + // Only define liveliness & readiness checks if container binds to a port + if defaultPort > 0 { + var handler corev1.Handler + if probePath != "" { + handler = corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: probePath, + Port: intstr.FromInt(defaultPort), + }, + } + } else { + // If no probe path is defined, assume app will bind to default TCP port + // TODO: handle apps that aren't socket listeners + handler = corev1.Handler{ TCPSocket: &corev1.TCPSocketAction{ Port: intstr.FromInt(defaultPort), }, - }, + } + } + + container.LivenessProbe = &corev1.Probe{ + Handler: handler, InitialDelaySeconds: initialDelaySeconds, TimeoutSeconds: timeoutSeconds, FailureThreshold: failureThreshold, - }, - ReadinessProbe: &corev1.Probe{ - Handler: corev1.Handler{ - TCPSocket: &corev1.TCPSocketAction{ - Port: intstr.FromInt(defaultPort), - }, - }, + } + container.ReadinessProbe = &corev1.Probe{ + Handler: handler, InitialDelaySeconds: initialDelaySeconds, TimeoutSeconds: timeoutSeconds, - }, - Env: env, - Resources: resourceRequirements, + } } - if p.config.Pod != nil && p.config.Pod.Container != nil { - containerCfg := p.config.Pod.Container - if containerCfg.Command != nil { - container.Command = *containerCfg.Command - } + return &container, nil +} - if containerCfg.Args != nil { - container.Args = *containerCfg.Args - } +// resourceDeploymentCreate creates the Kubernetes deployment. +func (p *Platform) resourceDeploymentCreate( + ctx context.Context, + log hclog.Logger, + src *component.Source, + img *docker.Image, + deployConfig *component.DeploymentConfig, + ui terminal.UI, + result *Deployment, + state *Resource_Deployment, + csinfo *clientsetInfo, + sg terminal.StepGroup, +) error { + // Prepare our namespace and override if set. + ns := csinfo.Namespace + if p.config.Namespace != "" { + ns = p.config.Namespace } - // Update the deployment with our spec - deployment.Spec.Template.Spec = corev1.PodSpec{ - Containers: []corev1.Container{container}, + step := sg.Add("") + defer func() { step.Abort() }() + step.Update("Kubernetes client connected to %s with namespace %s", csinfo.Config.Host, ns) + step.Done() + + step = sg.Add("Preparing deployment...") + + clientSet := csinfo.Clientset + deployClient := clientSet.AppsV1().Deployments(ns) + + // Determine if we have a deployment that we manage already + create := false + deployment, err := deployClient.Get(ctx, result.Name, metav1.GetOptions{}) + if errors.IsNotFound(err) { + deployment = result.newDeployment(result.Name) + create = true + err = nil + } + if err != nil { + return err } - // Override the default TCP socket checks if we have a probe path - if p.config.ProbePath != "" { - deployment.Spec.Template.Spec.Containers[0].LivenessProbe = &corev1.Probe{ - Handler: corev1.Handler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: p.config.ProbePath, - Port: intstr.FromInt(defaultPort), - }, - }, - InitialDelaySeconds: initialDelaySeconds, - TimeoutSeconds: timeoutSeconds, - FailureThreshold: failureThreshold, - } + // Setup our port configuration + if p.config.ServicePort == 0 && len(p.config.Ports) == 0 { + // nothing defined, set up the defaults + p.config.Ports = append(p.config.Ports, &Port{Port: DefaultServicePort, Name: "http"}) + } else if p.config.ServicePort > 0 && len(p.config.Ports) == 0 { + // old ServicePort var is used, so set it up in our Ports map to be used + p.config.Ports = append(p.config.Ports, &Port{Port: p.config.ServicePort, Name: "http"}) + } else if p.config.ServicePort > 0 && len(p.config.Ports) > 0 { + // both defined, this is an error + return fmt.Errorf("cannot define both 'service_port' and 'ports'. Use" + + " 'ports' for configuring multiple container ports") + } - deployment.Spec.Template.Spec.Containers[0].ReadinessProbe = &corev1.Probe{ - Handler: corev1.Handler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: p.config.ProbePath, - Port: intstr.FromInt(defaultPort), + envVars := make(map[string]string) + // Add deploy config environment to container env vars + for k, v := range p.config.StaticEnvVars { + envVars[k] = v + } + for k, v := range deployConfig.Env() { + envVars[k] = v + } + + // Check autoscaling + if p.config.AutoscaleConfig != nil && p.config.CPU == nil { + ui.Output("For autoscaling in Kubernetes to work, a deployment must specify "+ + "cpu resource limits and requests. Otherwise the metrics-server will not properly be able "+ + "to scale your deployment.", terminal.WithWarningStyle()) + } + + // Create scratch space volumes + var volumes []corev1.Volume + for idx := range p.config.ScratchSpace { + scratchName := fmt.Sprintf("scratch-%d", idx) + volumes = append(volumes, + corev1.Volume{ + Name: scratchName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, - InitialDelaySeconds: initialDelaySeconds, - TimeoutSeconds: timeoutSeconds, - FailureThreshold: failureThreshold, - } + ) } - if len(p.config.ScratchSpace) > 0 { - for idx, scratchSpaceLocation := range p.config.ScratchSpace { - scratchName := fmt.Sprintf("scratch-%d", idx) - deployment.Spec.Template.Spec.Volumes = append( - deployment.Spec.Template.Spec.Volumes, - corev1.Volume{ - Name: scratchName, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - ) + // Configure containers + var command *[]string + var args *[]string + if p.config.Pod != nil && p.config.Pod.Container != nil { + command = p.config.Pod.Container.Command + args = p.config.Pod.Container.Args + } + + appContainer, err := configureK8sContainer( + src.App, + fmt.Sprintf("%s:%s", img.Image, img.Tag), + p.config.Ports, + envVars, + p.config.Probe, + p.config.ProbePath, + p.config.CPU, + p.config.Memory, + p.config.Resources, + command, + args, + p.config.ScratchSpace, + volumes, + log, + ) + if err != nil { + return status.Errorf(status.Code(err), + "Failed to define app container: %s", err) + } - deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append( - deployment.Spec.Template.Spec.Containers[0].VolumeMounts, - corev1.VolumeMount{ - Name: scratchName, - MountPath: scratchSpaceLocation, - }, + var sidecarContainers []corev1.Container + if p.config.Pod != nil { + for _, sidecarConfig := range p.config.Pod.Sidecars { + sidecarEnvVars := make(map[string]string) + // Add deploy config environment to container env vars + for k, v := range sidecarConfig.StaticEnvVars { + sidecarEnvVars[k] = v + } + for k, v := range deployConfig.Env() { + sidecarEnvVars[k] = v + } + + sidecarContainer, err := configureK8sContainer( + sidecarConfig.Name, + sidecarConfig.Image, + sidecarConfig.Ports, + sidecarConfig.StaticEnvVars, + sidecarConfig.Probe, + sidecarConfig.ProbePath, + sidecarConfig.CPU, + sidecarConfig.Memory, + sidecarConfig.Resources, + sidecarConfig.Command, + sidecarConfig.Args, + p.config.ScratchSpace, + volumes, + log, ) + if err != nil { + return status.Errorf(status.Code(err), + "Failed to define sidecar container %s: %s", sidecarConfig.Name, err) + } + sidecarContainers = append(sidecarContainers, *sidecarContainer) } } + // Update the deployment with our spec + containers := []corev1.Container{*appContainer} + deployment.Spec.Template.Spec = corev1.PodSpec{ + Containers: append(containers, sidecarContainers...), + Volumes: volumes, + } + + // If no count is specified, presume that the user is managing the replica + // count some other way (perhaps manual scaling, perhaps a pod autoscaler). + // Either way if they don't specify a count, we should be sure we don't send one. + if p.config.Count > 0 { + deployment.Spec.Replicas = &p.config.Count + } + + // Set our ID on the label. We use this ID so that we can have a key + // to route to multiple versions during release management. + deployment.Spec.Template.Labels[labelId] = result.Id + + // Version label duplicates "labelId" to support services like Istio that + // expect pods to be labeled with 'version' + deployment.Spec.Template.Labels["version"] = result.Id + + // Apply user defined labels + for k, v := range p.config.Labels { + deployment.Spec.Template.Labels[k] = v + } + if p.config.Pod != nil { // Configure Pod podConfig := p.config.Pod @@ -617,11 +690,9 @@ func (p *Platform) resourceDeploymentCreate( } if p.config.ImageSecret != "" { - deployment.Spec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{ - { - Name: p.config.ImageSecret, - }, - } + deployment.Spec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{{ + Name: p.config.ImageSecret, + }} } if deployment.Spec.Template.Annotations == nil { @@ -663,6 +734,13 @@ func (p *Platform) resourceDeploymentCreate( dc := clientSet.AppsV1().Deployments(ns) + // TODO(izaak) delete me + j, err := json.Marshal(deployment) + if err != nil { + return fmt.Errorf("Failed to marshal deploy json") + } + log.Info(fmt.Sprintf("Deployment json \n\n %s \n\n", j)) + // Create/update if create { log.Debug("no existing deployment, creating a new one") @@ -674,7 +752,7 @@ func (p *Platform) resourceDeploymentCreate( deployment, err = dc.Update(ctx, deployment, metav1.UpdateOptions{}) } if err != nil { - return err + return status.Errorf(codes.Internal, "failed to create or update deployment: %s", err) } ev := clientSet.CoreV1().Events(ns) @@ -696,6 +774,24 @@ func (p *Platform) resourceDeploymentCreate( reportedError bool ) + var timeoutSeconds int + var failureThreshold int + var initialDelaySeconds int + + for _, container := range deployment.Spec.Template.Spec.Containers { + if int(container.ReadinessProbe.TimeoutSeconds) > timeoutSeconds { + timeoutSeconds = int(container.ReadinessProbe.TimeoutSeconds) + } + + if int(container.ReadinessProbe.FailureThreshold) > failureThreshold { + failureThreshold = int(container.ReadinessProbe.FailureThreshold) + } + + if int(container.ReadinessProbe.TimeoutSeconds) > initialDelaySeconds { + initialDelaySeconds = int(container.ReadinessProbe.InitialDelaySeconds) + } + } + // We wait the maximum amount of time that the deployment controller would wait for a pod // to start before exiting. We double the time to allow for various Kubernetes based // delays in startup, detection, and reporting. @@ -1239,7 +1335,8 @@ type Config struct { // A full resource of options to define ports for your service running on the container // Defaults to port 3000. - Ports []map[string]string `hcl:"ports,optional"` + // Todo(XX): add in HCL parse logic to warn if defining ports the old way, & update docs + Ports []*Port `hcl:"port,block"` // If set, this is the HTTP path to request to test that the application // is up and running. Without this, we only test that a connection can be @@ -1303,6 +1400,30 @@ type AutoscaleConfig struct { type Pod struct { SecurityContext *PodSecurityContext `hcl:"security_context,block"` Container *Container `hcl:"container,block"` + Sidecars []*SidecarContainer `hcl:"sidecar,block"` +} + +// SidecarContainer describes the configuration for the sidecar container +type SidecarContainer struct { + Name string `hcl:"name"` + Image string `hcl:"image"` + Ports []*Port `hcl:"port,block"` + ProbePath string `hcl:"probe_path,optional"` + Probe *Probe `hcl:"probe,block"` + CPU *ResourceConfig `hcl:"cpu,block"` + Memory *ResourceConfig `hcl:"memory,block"` + Resources map[string]string `hcl:"resources,optional"` + Command *[]string `hcl:"command,optional"` + Args *[]string `hcl:"args,optional"` + StaticEnvVars map[string]string `hcl:"static_environment,optional"` +} + +type Port struct { + Name string `hcl:"name"` + Port uint `hcl:"port"` + HostPort uint `hcl:"host_port,optional"` + HostIP string `hcl:"host_ip,optional"` + Protocol string `hcl:"protocol,optional"` } // Container describes the commands and arguments for a container config @@ -1488,12 +1609,11 @@ deploy "kubernetes" { ) doc.SetField( - "ports", - "a map of ports and options that the application is listening on", + "port", + "a port and options that the application is listening on", docs.Summary( "used to define and expose multiple ports that the application is", - "listening on for the container in use. Available keys are 'port', 'name'", - ", 'host_port', and 'host_ip'. Ports defined will be TCP protocol.", + "listening on for the container in use. Can be specified multiple times for many ports.", ), ) From 9a3e90306105cdcb206b6da363f5d2cbd1a818ec Mon Sep 17 00:00:00 2001 From: Shirley Xiaolin Xu Date: Wed, 6 Oct 2021 15:07:32 -0400 Subject: [PATCH 02/13] changelog --- .changelog/2428.txt | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changelog/2428.txt diff --git a/.changelog/2428.txt b/.changelog/2428.txt new file mode 100644 index 00000000000..24d78c42fa8 --- /dev/null +++ b/.changelog/2428.txt @@ -0,0 +1,8 @@ +```release-note:feature +plugin/k8s: Allows advanced users to add sidecar containers to apps using the k8s plugin config. +``` + +``` +release-note:breaking-change +plugin/k8s: "Ports" config is now []*Port rather than map[string]string +``` \ No newline at end of file From e29cc37ef0326f0209cec60caa975c7e50daf584 Mon Sep 17 00:00:00 2001 From: Izaak Lauer <8404559+izaaklauer@users.noreply.github.com> Date: Wed, 6 Oct 2021 15:48:21 -0400 Subject: [PATCH 03/13] Using the official docker parser Which I feel like was harder to discover how to use than it needs to be --- builtin/k8s/platform.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/builtin/k8s/platform.go b/builtin/k8s/platform.go index 392cefd9e44..f12cb94d571 100644 --- a/builtin/k8s/platform.go +++ b/builtin/k8s/platform.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/docker/distribution/reference" "github.com/hashicorp/go-hclog" "github.com/mitchellh/mapstructure" "google.golang.org/grpc/codes" @@ -307,22 +308,19 @@ func configureK8sContainer( ) (*corev1.Container, error) { // If the user is using the latest tag, then don't specify an overriding pull policy. // This by default means kubernetes will always pull so that latest is useful. - splitImage := strings.Split(image, ":") + var pullPolicy corev1.PullPolicy - if len(splitImage) == 1 { + imageReference, err := reference.Parse(image) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "image %q is not a valid OCI reference: %q", image, err) + } + taggedImageReference, ok := imageReference.(reference.Tagged) + if !ok || taggedImageReference.Tag() == "latest" { // If no tag is set, docker will default to "latest", and we always want to pull. pullPolicy = corev1.PullAlways - } else if len(splitImage) == 2 { - tag := splitImage[1] - if tag == "latest" { - // Always pull a latest image, so the k8s workers don't use their local cache and ignore new changes. - pullPolicy = corev1.PullAlways - } else { - // A tag is present, we can use k8s worker caching - pullPolicy = corev1.PullIfNotPresent - } } else { - return nil, status.Errorf(codes.InvalidArgument, "image %s is in an invalid format", image) + // A tag is present, we can use k8s worker caching + pullPolicy = corev1.PullIfNotPresent } var k8sPorts []corev1.ContainerPort From 43b349d1b7bb43c7879453bbd6ed66cd26a3eff9 Mon Sep 17 00:00:00 2001 From: Izaak Lauer <8404559+izaaklauer@users.noreply.github.com> Date: Wed, 6 Oct 2021 17:21:07 -0400 Subject: [PATCH 04/13] Refactoring config to have a single container definition That is reused between the app container and sidecars --- builtin/k8s/platform.go | 232 ++++++++++++++++++++++++---------------- 1 file changed, 141 insertions(+), 91 deletions(-) diff --git a/builtin/k8s/platform.go b/builtin/k8s/platform.go index f12cb94d571..b7f79a7ad3f 100644 --- a/builtin/k8s/platform.go +++ b/builtin/k8s/platform.go @@ -7,6 +7,8 @@ import ( "strings" "time" + "github.com/mitchellh/copystructure" + "github.com/docker/distribution/reference" "github.com/hashicorp/go-hclog" "github.com/mitchellh/mapstructure" @@ -291,17 +293,8 @@ func (p *Platform) resourceDeploymentStatus( } func configureK8sContainer( - name string, - image string, - ports []*Port, + c *Container, envVars map[string]string, - probe *Probe, - probePath string, - cpu *ResourceConfig, - memory *ResourceConfig, - resources map[string]string, - command *[]string, - args *[]string, scratchSpace []string, volumes []corev1.Volume, log hclog.Logger, @@ -310,9 +303,9 @@ func configureK8sContainer( // This by default means kubernetes will always pull so that latest is useful. var pullPolicy corev1.PullPolicy - imageReference, err := reference.Parse(image) + imageReference, err := reference.Parse(c.Image) if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "image %q is not a valid OCI reference: %q", image, err) + return nil, status.Errorf(codes.InvalidArgument, "image %q is not a valid OCI reference: %q", c.Image, err) } taggedImageReference, ok := imageReference.(reference.Tagged) if !ok || taggedImageReference.Tag() == "latest" { @@ -324,7 +317,7 @@ func configureK8sContainer( } var k8sPorts []corev1.ContainerPort - for _, port := range ports { + for _, port := range c.Ports { k8sPorts = append(k8sPorts, corev1.ContainerPort{ Name: port.Name, ContainerPort: int32(port.Port), @@ -353,15 +346,15 @@ func configureK8sContainer( timeoutSeconds := int32(5) failureThreshold := int32(5) - if probe != nil { - if probe.InitialDelaySeconds != 0 { - initialDelaySeconds = int32(probe.InitialDelaySeconds) + if c.Probe != nil { + if c.Probe.InitialDelaySeconds != 0 { + initialDelaySeconds = int32(c.Probe.InitialDelaySeconds) } - if probe.TimeoutSeconds != 0 { - timeoutSeconds = int32(probe.TimeoutSeconds) + if c.Probe.TimeoutSeconds != 0 { + timeoutSeconds = int32(c.Probe.TimeoutSeconds) } - if probe.FailureThreshold != 0 { - failureThreshold = int32(probe.FailureThreshold) + if c.Probe.FailureThreshold != 0 { + failureThreshold = int32(c.Probe.FailureThreshold) } } @@ -369,47 +362,47 @@ func configureK8sContainer( var resourceLimits = make(map[corev1.ResourceName]k8sresource.Quantity) var resourceRequests = make(map[corev1.ResourceName]k8sresource.Quantity) - if cpu != nil { - if cpu.Requested != "" { - q, err := k8sresource.ParseQuantity(cpu.Requested) + if c.CPU != nil { + if c.CPU.Requested != "" { + q, err := k8sresource.ParseQuantity(c.CPU.Requested) if err != nil { return nil, - status.Errorf(codes.InvalidArgument, "failed to parse cpu request %s to k8s quantity: %s", cpu.Requested, err) + status.Errorf(codes.InvalidArgument, "failed to parse cpu request %s to k8s quantity: %s", c.CPU.Requested, err) } resourceRequests[corev1.ResourceCPU] = q } - if cpu.Limit != "" { - q, err := k8sresource.ParseQuantity(cpu.Limit) + if c.CPU.Limit != "" { + q, err := k8sresource.ParseQuantity(c.CPU.Limit) if err != nil { return nil, - status.Errorf(codes.InvalidArgument, "failed to parse cpu limit %s to k8s quantity: %s", cpu.Limit, err) + status.Errorf(codes.InvalidArgument, "failed to parse cpu limit %s to k8s quantity: %s", c.CPU.Limit, err) } resourceLimits[corev1.ResourceCPU] = q } } - if memory != nil { - if memory.Requested != "" { - q, err := k8sresource.ParseQuantity(memory.Requested) + if c.Memory != nil { + if c.Memory.Requested != "" { + q, err := k8sresource.ParseQuantity(c.Memory.Requested) if err != nil { return nil, - status.Errorf(codes.InvalidArgument, "failed to parse memory requested %s to k8s quantity: %s", memory.Requested, err) + status.Errorf(codes.InvalidArgument, "failed to parse memory requested %s to k8s quantity: %s", c.Memory.Requested, err) } resourceRequests[corev1.ResourceMemory] = q } - if memory.Limit != "" { - q, err := k8sresource.ParseQuantity(memory.Limit) + if c.Memory.Limit != "" { + q, err := k8sresource.ParseQuantity(c.Memory.Limit) if err != nil { return nil, - status.Errorf(codes.InvalidArgument, "failed to parse memory limit %s to k8s quantity: %s", memory.Limit, err) + status.Errorf(codes.InvalidArgument, "failed to parse memory limit %s to k8s quantity: %s", c.Memory.Limit, err) } resourceLimits[corev1.ResourceMemory] = q } } - for k, v := range resources { + for k, v := range c.Resources { if strings.HasPrefix(k, "limits_") || strings.HasPrefix(k, "requests_") { key := strings.Split(k, "_") resourceName := corev1.ResourceName(key[1]) @@ -443,8 +436,8 @@ func configureK8sContainer( } container := corev1.Container{ - Name: name, - Image: image, + Name: c.Name, + Image: c.Image, ImagePullPolicy: pullPolicy, Env: k8sEnvVars, Resources: resourceRequirements, @@ -454,20 +447,20 @@ func configureK8sContainer( if len(k8sPorts) > 0 { container.Ports = k8sPorts } - if command != nil { - container.Command = *command + if c.Command != nil { + container.Command = *c.Command } - if args != nil { - container.Args = *args + if c.Args != nil { + container.Args = *c.Args } // Only define liveliness & readiness checks if container binds to a port if defaultPort > 0 { var handler corev1.Handler - if probePath != "" { + if c.ProbePath != "" { handler = corev1.Handler{ HTTPGet: &corev1.HTTPGetAction{ - Path: probePath, + Path: c.ProbePath, Port: intstr.FromInt(defaultPort), }, } @@ -538,6 +531,13 @@ func (p *Platform) resourceDeploymentCreate( return err } + var overlayTarget *Container + if p.config.Pod != nil && p.config.Pod.Container != nil { + overlayTarget = p.config.Pod.Container + } else { + overlayTarget = &Container{} + } + // Setup our port configuration if p.config.ServicePort == 0 && len(p.config.Ports) == 0 { // nothing defined, set up the defaults @@ -551,6 +551,11 @@ func (p *Platform) resourceDeploymentCreate( " 'ports' for configuring multiple container ports") } + appContainerSpec, err := overlayTopLevelProperties(p.config, overlayTarget) + if err != nil { + return status.Errorf(codes.InvalidArgument, "Failed to parse container config: %s", err) + } + envVars := make(map[string]string) // Add deploy config environment to container env vars for k, v := range p.config.StaticEnvVars { @@ -581,26 +586,12 @@ func (p *Platform) resourceDeploymentCreate( ) } - // Configure containers - var command *[]string - var args *[]string - if p.config.Pod != nil && p.config.Pod.Container != nil { - command = p.config.Pod.Container.Command - args = p.config.Pod.Container.Args - } + appContainerSpec.Image = fmt.Sprintf("%s:%s", img.Image, img.Tag) + appContainerSpec.Name = src.App appContainer, err := configureK8sContainer( - src.App, - fmt.Sprintf("%s:%s", img.Image, img.Tag), - p.config.Ports, + appContainerSpec, envVars, - p.config.Probe, - p.config.ProbePath, - p.config.CPU, - p.config.Memory, - p.config.Resources, - command, - args, p.config.ScratchSpace, volumes, log, @@ -613,27 +604,18 @@ func (p *Platform) resourceDeploymentCreate( var sidecarContainers []corev1.Container if p.config.Pod != nil { for _, sidecarConfig := range p.config.Pod.Sidecars { - sidecarEnvVars := make(map[string]string) + envVars := make(map[string]string) // Add deploy config environment to container env vars for k, v := range sidecarConfig.StaticEnvVars { - sidecarEnvVars[k] = v + envVars[k] = v } for k, v := range deployConfig.Env() { - sidecarEnvVars[k] = v + envVars[k] = v } sidecarContainer, err := configureK8sContainer( - sidecarConfig.Name, - sidecarConfig.Image, - sidecarConfig.Ports, - sidecarConfig.StaticEnvVars, - sidecarConfig.Probe, - sidecarConfig.ProbePath, - sidecarConfig.CPU, - sidecarConfig.Memory, - sidecarConfig.Resources, - sidecarConfig.Command, - sidecarConfig.Args, + sidecarConfig, + envVars, p.config.ScratchSpace, volumes, log, @@ -1296,6 +1278,80 @@ func (p *Platform) Status( return result, nil } +// overlayDefaultProperties overlays the top level container properties from config onto the +// more detailed container properties in container, erroring if both are not empty. +func overlayTopLevelProperties(config Config, container *Container) (*Container, error) { + var overlaidContainer *Container + i, err := copystructure.Copy(container) + if err != nil { + return nil, status.Errorf(codes.Internal, "Failed to initialize container spec: %s", err) + } + overlaidContainer, _ = i.(*Container) + + // Layer in easily discoverable top-level properties, and error on conflicts + if config.ProbePath != "" { + if container.ProbePath != "" { + return nil, status.Errorf(codes.InvalidArgument, "ProbePath defined multiple times - in top-level config and in Pod.Container.") + } + overlaidContainer.ProbePath = config.ProbePath + } + + if config.Probe != nil { + if container.Probe != nil { + return nil, status.Errorf(codes.InvalidArgument, "Probe defined multiple times - in top-level config and in Pod.Container.") + } + overlaidContainer.ProbePath = config.ProbePath + } + + if config.Resources != nil { + if container.Resources != nil { + return nil, status.Errorf(codes.InvalidArgument, "Resources defined multiple times - in top-level config and in Pod.Container.") + } + overlaidContainer.Resources = config.Resources + } + + if config.CPU != nil { + if container.CPU != nil { + return nil, status.Errorf(codes.InvalidArgument, "CPU defined multiple times - in top-level config and in Pod.Container.") + } + overlaidContainer.CPU = config.CPU + } + + if config.Memory != nil { + if container.Memory != nil { + return nil, status.Errorf(codes.InvalidArgument, "Memory defined multiple times - in top-level config and in Pod.Container.") + } + overlaidContainer.Memory = config.Memory + } + + if config.StaticEnvVars != nil { + if container.StaticEnvVars != nil { + return nil, status.Errorf(codes.InvalidArgument, "StaticEnvVars defined multiple times - in top-level config and in Pod.Container.") + } + overlaidContainer.StaticEnvVars = config.StaticEnvVars + } + + if config.StaticEnvVars != nil { + if container.StaticEnvVars != nil { + return nil, status.Errorf(codes.InvalidArgument, "StaticEnvVars defined multiple times - in top-level config and in Pod.Container.") + } + overlaidContainer.StaticEnvVars = config.StaticEnvVars + } + + if config.Ports != nil { + if container.Ports != nil { + if config.ServicePort != 0 { + return nil, status.Errorf(codes.InvalidArgument, "Ports defined multiple times - in top-level config as ServicePort and in Pod.Container.") + } else { + return nil, status.Errorf(codes.InvalidArgument, "Ports defined multiple times - in top-level config and in Pod.Container.") + } + } + overlaidContainer.Ports = config.Ports + } + + return overlaidContainer, nil +} + // Config is the configuration structure for the Platform. type Config struct { // Annotations are added to the pod spec of the deployed application. This is @@ -1398,11 +1454,19 @@ type AutoscaleConfig struct { type Pod struct { SecurityContext *PodSecurityContext `hcl:"security_context,block"` Container *Container `hcl:"container,block"` - Sidecars []*SidecarContainer `hcl:"sidecar,block"` + Sidecars []*Container `hcl:"sidecar,block"` +} + +type Port struct { + Name string `hcl:"name"` + Port uint `hcl:"port"` + HostPort uint `hcl:"host_port,optional"` + HostIP string `hcl:"host_ip,optional"` + Protocol string `hcl:"protocol,optional"` } -// SidecarContainer describes the configuration for the sidecar container -type SidecarContainer struct { +// Container describes the detailed parameters to declare a kubernetes container +type Container struct { Name string `hcl:"name"` Image string `hcl:"image"` Ports []*Port `hcl:"port,block"` @@ -1416,20 +1480,6 @@ type SidecarContainer struct { StaticEnvVars map[string]string `hcl:"static_environment,optional"` } -type Port struct { - Name string `hcl:"name"` - Port uint `hcl:"port"` - HostPort uint `hcl:"host_port,optional"` - HostIP string `hcl:"host_ip,optional"` - Protocol string `hcl:"protocol,optional"` -} - -// Container describes the commands and arguments for a container config -type Container struct { - Command *[]string `hcl:"command"` - Args *[]string `hcl:"args"` -} - // PodSecurityContext describes the security config for the Pod type PodSecurityContext struct { RunAsUser *int64 `hcl:"run_as_user"` From 5b21a811f3e8628c3a3b145739c0c0ae94f78dfd Mon Sep 17 00:00:00 2001 From: Shirley Xiaolin Xu Date: Thu, 7 Oct 2021 13:59:41 -0400 Subject: [PATCH 05/13] website --- .../components/platform-kubernetes.mdx | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/website/content/partials/components/platform-kubernetes.mdx b/website/content/partials/components/platform-kubernetes.mdx index b103b65384b..d441ccfef1f 100644 --- a/website/content/partials/components/platform-kubernetes.mdx +++ b/website/content/partials/components/platform-kubernetes.mdx @@ -123,6 +123,18 @@ The UID to run the entrypoint of the container process. - Type: **k8s.PodSecurityContext** +##### pod.sidecar + +- Type: **list of k8s.Container** + +#### port + +A port and options that the application is listening on. + +Used to define and expose multiple ports that the application is listening on for the container in use. Can be specified multiple times for many ports. + +- Type: **list of k8s.Port** + #### probe (category) Configuration to control liveness and readiness probes. @@ -209,15 +221,6 @@ Namespace is the name of the Kubernetes namespace to apply the deployment in. Th - Type: **string** - **Optional** -#### ports - -A map of ports and options that the application is listening on. - -Used to define and expose multiple ports that the application is listening on for the container in use. Available keys are 'port', 'name' , 'host_port', and 'host_ip'. Ports defined will be TCP protocol. - -- Type: **list of map of string to string** -- **Optional** - #### probe_path The HTTP path to request to test that the application is running. From 7183ee3062df75a56bf9d2402fcbdd3552e9e867 Mon Sep 17 00:00:00 2001 From: Shirley Xiaolin Xu Date: Thu, 7 Oct 2021 17:05:24 -0400 Subject: [PATCH 06/13] move crashloop check --- builtin/k8s/platform.go | 53 ++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/builtin/k8s/platform.go b/builtin/k8s/platform.go index b7f79a7ad3f..914fb2e1984 100644 --- a/builtin/k8s/platform.go +++ b/builtin/k8s/platform.go @@ -143,7 +143,6 @@ func (p *Platform) getClientset() (*clientsetInfo, error) { func (p *Platform) resourceDeploymentStatus( ctx context.Context, - log hclog.Logger, sg terminal.StepGroup, deploymentState *Resource_Deployment, clientset *clientsetInfo, @@ -297,7 +296,9 @@ func configureK8sContainer( envVars map[string]string, scratchSpace []string, volumes []corev1.Volume, + autoscaleConfig *AutoscaleConfig, log hclog.Logger, + ui terminal.UI, ) (*corev1.Container, error) { // If the user is using the latest tag, then don't specify an overriding pull policy. // This by default means kubernetes will always pull so that latest is useful. @@ -403,21 +404,39 @@ func configureK8sContainer( } for k, v := range c.Resources { - if strings.HasPrefix(k, "limits_") || strings.HasPrefix(k, "requests_") { - key := strings.Split(k, "_") - resourceName := corev1.ResourceName(key[1]) + if strings.HasPrefix(k, "limits_") { + limitKey := strings.Split(k, "_") + resourceName := corev1.ResourceName(limitKey[1]) - quantity, err := k8sresource.ParseQuantity(v) + q, err := k8sresource.ParseQuantity(v) if err != nil { - return nil, - status.Errorf(codes.InvalidArgument, "failed to parse resource %s to k8s quantity: %s", v, err) + return nil, status.Errorf(codes.InvalidArgument, "failed to parse resource %s to k8s quantity: %s", v, err) + } + resourceLimits[resourceName] = q + } else if strings.HasPrefix(k, "requests_") { + reqKey := strings.Split(k, "_") + resourceName := corev1.ResourceName(reqKey[1]) + + q, err := k8sresource.ParseQuantity(v) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "failed to parse resource %s to k8s quantity: %s", v, err) } - resourceLimits[resourceName] = quantity + resourceRequests[resourceName] = q } else { log.Warn("ignoring unrecognized k8s resources key: %q", k) } } + _, cpuLimit := resourceLimits[corev1.ResourceCPU] + _, cpuRequest := resourceRequests[corev1.ResourceCPU] + + // Check autoscaling + if autoscaleConfig != nil && !(cpuLimit || cpuRequest) { + ui.Output("For autoscaling in Kubernetes to work, a deployment must specify "+ + "cpu resource limits and requests. Otherwise the metrics-server will not properly be able "+ + "to scale your deployment.", terminal.WithWarningStyle()) + } + resourceRequirements := corev1.ResourceRequirements{ Limits: resourceLimits, Requests: resourceRequests, @@ -565,13 +584,6 @@ func (p *Platform) resourceDeploymentCreate( envVars[k] = v } - // Check autoscaling - if p.config.AutoscaleConfig != nil && p.config.CPU == nil { - ui.Output("For autoscaling in Kubernetes to work, a deployment must specify "+ - "cpu resource limits and requests. Otherwise the metrics-server will not properly be able "+ - "to scale your deployment.", terminal.WithWarningStyle()) - } - // Create scratch space volumes var volumes []corev1.Volume for idx := range p.config.ScratchSpace { @@ -594,7 +606,9 @@ func (p *Platform) resourceDeploymentCreate( envVars, p.config.ScratchSpace, volumes, + p.config.AutoscaleConfig, log, + ui, ) if err != nil { return status.Errorf(status.Code(err), @@ -618,7 +632,9 @@ func (p *Platform) resourceDeploymentCreate( envVars, p.config.ScratchSpace, volumes, + p.config.AutoscaleConfig, log, + ui, ) if err != nil { return status.Errorf(status.Code(err), @@ -714,13 +730,6 @@ func (p *Platform) resourceDeploymentCreate( dc := clientSet.AppsV1().Deployments(ns) - // TODO(izaak) delete me - j, err := json.Marshal(deployment) - if err != nil { - return fmt.Errorf("Failed to marshal deploy json") - } - log.Info(fmt.Sprintf("Deployment json \n\n %s \n\n", j)) - // Create/update if create { log.Debug("no existing deployment, creating a new one") From b5d3fbf056cd16ba78418482c654e3a4cc96b90d Mon Sep 17 00:00:00 2001 From: Izaak Lauer <8404559+izaaklauer@users.noreply.github.com> Date: Mon, 11 Oct 2021 21:35:57 -0400 Subject: [PATCH 07/13] Fixing up config for sidecars - Adding validators - Adding field docs - Hard deprecating top level Ports, and adding a helpful error --- builtin/k8s/platform.go | 585 ++++++++++-------- go.mod | 2 +- go.sum | 4 +- internal/cli/app_docs.go | 7 + .../components/platform-kubernetes.mdx | 240 ++++++- 5 files changed, 581 insertions(+), 257 deletions(-) diff --git a/builtin/k8s/platform.go b/builtin/k8s/platform.go index 914fb2e1984..c116ea1f313 100644 --- a/builtin/k8s/platform.go +++ b/builtin/k8s/platform.go @@ -7,10 +7,11 @@ import ( "strings" "time" - "github.com/mitchellh/copystructure" - "github.com/docker/distribution/reference" + validation "github.com/go-ozzo/ozzo-validation/v4" "github.com/hashicorp/go-hclog" + "github.com/hashicorp/waypoint/builtin/aws/utils" + "github.com/mitchellh/copystructure" "github.com/mitchellh/mapstructure" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -38,7 +39,6 @@ const ( labelId = "waypoint.hashicorp.com/id" labelNonce = "waypoint.hashicorp.com/nonce" - // TODO Evaluate if this should remain as a default 3000 to another port. DefaultServicePort = 3000 ) @@ -99,6 +99,57 @@ func (p *Platform) DefaultReleaserFunc() interface{} { } } +// ConfigSet is called after a configuration has been decoded +// we can use this to validate the config +func (p *Platform) ConfigSet(config interface{}) error { + c, ok := config.(*Config) + if !ok { + // this should never happen + return status.Errorf(codes.FailedPrecondition, "invalid configuration, expected *k8s.Config, got %T", config) + } + + if len(c.DeprecatedPorts) > 0 { + return status.Errorf(codes.InvalidArgument, "invalid kubernetes platform config - the 'ports' field has been deprecated and removed "+ + "in favor of Pod.Container.Port. Refer to Port documentation here: https://www.waypointproject.io/plugins/kubernetes#port") + } + + // Some fields can be specified on pod.Container and at the top level, for convenience and for + // historical reasons. Validate that both are not set at once. + if c.Pod != nil && c.Pod.Container != nil { + containerOverlayErrStr := "%s defined multiple times - in top level config and in Pod.Container" + container := c.Pod.Container + err := utils.Error(validation.ValidateStruct(c, + validation.Field(&c.Probe, + validation.Empty.When(container.Probe != nil).Error(fmt.Sprintf(containerOverlayErrStr, "Probe")), + ), + validation.Field(&c.ProbePath, + validation.Empty.When(container.ProbePath != "").Error(fmt.Sprintf(containerOverlayErrStr, "ProbePath")), + ), + validation.Field(&c.Resources, + validation.Empty.When(container.Resources != nil).Error(fmt.Sprintf(containerOverlayErrStr, "Resources")), + ), + validation.Field(&c.CPU, + validation.Empty.When(container.CPU != nil).Error(fmt.Sprintf(containerOverlayErrStr, "CPU")), + ), + validation.Field(&c.Memory, + validation.Empty.When(container.Memory != nil).Error(fmt.Sprintf(containerOverlayErrStr, "Memory")), + ), + validation.Field(&c.StaticEnvVars, + validation.Empty.When(container.StaticEnvVars != nil).Error(fmt.Sprintf(containerOverlayErrStr, "StaticEnvVars")), + ), + validation.Field(&c.ServicePort, + validation.Empty.When(len(container.Ports) > 0).Error("Cannot define both 'service_port' and container 'port'. Use"+ + " container 'port' multiple times for configuring multiple container ports"), + ), + )) + if err != nil { + return status.Errorf(codes.InvalidArgument, fmt.Sprintf("Invalid kubernetes platform plugin config: %s", err.Error())) + } + } + + return nil +} + func (p *Platform) resourceManager(log hclog.Logger, dcr *component.DeclaredResourcesResp) *resource.Manager { return resource.NewManager( resource.WithLogger(log.Named("resource_manager")), @@ -291,8 +342,9 @@ func (p *Platform) resourceDeploymentStatus( return nil } -func configureK8sContainer( +func configureContainer( c *Container, + image string, envVars map[string]string, scratchSpace []string, volumes []corev1.Volume, @@ -304,9 +356,9 @@ func configureK8sContainer( // This by default means kubernetes will always pull so that latest is useful. var pullPolicy corev1.PullPolicy - imageReference, err := reference.Parse(c.Image) + imageReference, err := reference.Parse(image) if err != nil { - return nil, status.Errorf(codes.InvalidArgument, "image %q is not a valid OCI reference: %q", c.Image, err) + return nil, status.Errorf(codes.InvalidArgument, "image %q is not a valid OCI reference: %q", image, err) } taggedImageReference, ok := imageReference.(reference.Tagged) if !ok || taggedImageReference.Tag() == "latest" { @@ -319,6 +371,11 @@ func configureK8sContainer( var k8sPorts []corev1.ContainerPort for _, port := range c.Ports { + // Default the port protocol to TCP + if port.Protocol == "" { + port.Protocol = "TCP" + } + k8sPorts = append(k8sPorts, corev1.ContainerPort{ Name: port.Name, ContainerPort: int32(port.Port), @@ -456,7 +513,7 @@ func configureK8sContainer( container := corev1.Container{ Name: c.Name, - Image: c.Image, + Image: image, ImagePullPolicy: pullPolicy, Env: k8sEnvVars, Resources: resourceRequirements, @@ -557,24 +614,17 @@ func (p *Platform) resourceDeploymentCreate( overlayTarget = &Container{} } - // Setup our port configuration - if p.config.ServicePort == 0 && len(p.config.Ports) == 0 { - // nothing defined, set up the defaults - p.config.Ports = append(p.config.Ports, &Port{Port: DefaultServicePort, Name: "http"}) - } else if p.config.ServicePort > 0 && len(p.config.Ports) == 0 { - // old ServicePort var is used, so set it up in our Ports map to be used - p.config.Ports = append(p.config.Ports, &Port{Port: p.config.ServicePort, Name: "http"}) - } else if p.config.ServicePort > 0 && len(p.config.Ports) > 0 { - // both defined, this is an error - return fmt.Errorf("cannot define both 'service_port' and 'ports'. Use" + - " 'ports' for configuring multiple container ports") - } - appContainerSpec, err := overlayTopLevelProperties(p.config, overlayTarget) if err != nil { return status.Errorf(codes.InvalidArgument, "Failed to parse container config: %s", err) } + // App container must have some kind of port + if len(appContainerSpec.Ports) == 0 { + ui.Output(fmt.Sprintf("No ports defined - defaulting to http on port %d", DefaultServicePort), terminal.WithWarningStyle()) + appContainerSpec.Ports = append(appContainerSpec.Ports, &Port{Port: DefaultServicePort, Name: "http"}) + } + envVars := make(map[string]string) // Add deploy config environment to container env vars for k, v := range p.config.StaticEnvVars { @@ -598,11 +648,12 @@ func (p *Platform) resourceDeploymentCreate( ) } - appContainerSpec.Image = fmt.Sprintf("%s:%s", img.Image, img.Tag) + appImage := fmt.Sprintf("%s:%s", img.Image, img.Tag) appContainerSpec.Name = src.App - appContainer, err := configureK8sContainer( + appContainer, err := configureContainer( appContainerSpec, + appImage, envVars, p.config.ScratchSpace, volumes, @@ -620,15 +671,16 @@ func (p *Platform) resourceDeploymentCreate( for _, sidecarConfig := range p.config.Pod.Sidecars { envVars := make(map[string]string) // Add deploy config environment to container env vars - for k, v := range sidecarConfig.StaticEnvVars { + for k, v := range sidecarConfig.Container.StaticEnvVars { envVars[k] = v } for k, v := range deployConfig.Env() { envVars[k] = v } - sidecarContainer, err := configureK8sContainer( - sidecarConfig, + sidecarContainer, err := configureContainer( + &sidecarConfig.Container, + sidecarConfig.Image, envVars, p.config.ScratchSpace, volumes, @@ -638,7 +690,7 @@ func (p *Platform) resourceDeploymentCreate( ) if err != nil { return status.Errorf(status.Code(err), - "Failed to define sidecar container %s: %s", sidecarConfig.Name, err) + "Failed to define sidecar container %s: %s", sidecarConfig.Container.Name, err) } sidecarContainers = append(sidecarContainers, *sidecarContainer) } @@ -1288,7 +1340,8 @@ func (p *Platform) Status( } // overlayDefaultProperties overlays the top level container properties from config onto the -// more detailed container properties in container, erroring if both are not empty. +// more detailed container properties in container. +// ConfigSet has already validated that both are not set, so we don't have to check here. func overlayTopLevelProperties(config Config, container *Container) (*Container, error) { var overlaidContainer *Container i, err := copystructure.Copy(container) @@ -1297,65 +1350,27 @@ func overlayTopLevelProperties(config Config, container *Container) (*Container, } overlaidContainer, _ = i.(*Container) - // Layer in easily discoverable top-level properties, and error on conflicts if config.ProbePath != "" { - if container.ProbePath != "" { - return nil, status.Errorf(codes.InvalidArgument, "ProbePath defined multiple times - in top-level config and in Pod.Container.") - } overlaidContainer.ProbePath = config.ProbePath } - if config.Probe != nil { - if container.Probe != nil { - return nil, status.Errorf(codes.InvalidArgument, "Probe defined multiple times - in top-level config and in Pod.Container.") - } overlaidContainer.ProbePath = config.ProbePath } - if config.Resources != nil { - if container.Resources != nil { - return nil, status.Errorf(codes.InvalidArgument, "Resources defined multiple times - in top-level config and in Pod.Container.") - } overlaidContainer.Resources = config.Resources } - if config.CPU != nil { - if container.CPU != nil { - return nil, status.Errorf(codes.InvalidArgument, "CPU defined multiple times - in top-level config and in Pod.Container.") - } overlaidContainer.CPU = config.CPU } - if config.Memory != nil { - if container.Memory != nil { - return nil, status.Errorf(codes.InvalidArgument, "Memory defined multiple times - in top-level config and in Pod.Container.") - } overlaidContainer.Memory = config.Memory } - if config.StaticEnvVars != nil { - if container.StaticEnvVars != nil { - return nil, status.Errorf(codes.InvalidArgument, "StaticEnvVars defined multiple times - in top-level config and in Pod.Container.") - } overlaidContainer.StaticEnvVars = config.StaticEnvVars } - - if config.StaticEnvVars != nil { - if container.StaticEnvVars != nil { - return nil, status.Errorf(codes.InvalidArgument, "StaticEnvVars defined multiple times - in top-level config and in Pod.Container.") - } - overlaidContainer.StaticEnvVars = config.StaticEnvVars - } - - if config.Ports != nil { - if container.Ports != nil { - if config.ServicePort != 0 { - return nil, status.Errorf(codes.InvalidArgument, "Ports defined multiple times - in top-level config as ServicePort and in Pod.Container.") - } else { - return nil, status.Errorf(codes.InvalidArgument, "Ports defined multiple times - in top-level config and in Pod.Container.") - } - } - overlaidContainer.Ports = config.Ports + if config.ServicePort != nil { + // We've already validated that ports is nil in ConfigSet - they cannot both be set at once. + container.Ports = []*Port{{Port: *config.ServicePort, Name: "http"}} } return overlaidContainer, nil @@ -1396,11 +1411,6 @@ type Config struct { // Namespace is the Kubernetes namespace to target the deployment to. Namespace string `hcl:"namespace,optional"` - // A full resource of options to define ports for your service running on the container - // Defaults to port 3000. - // Todo(XX): add in HCL parse logic to warn if defining ports the old way, & update docs - Ports []*Port `hcl:"port,block"` - // If set, this is the HTTP path to request to test that the application // is up and running. Without this, we only test that a connection can be // made to the port. @@ -1430,16 +1440,19 @@ type Config struct { // Port that your service is running on within the actual container. // Defaults to DefaultServicePort const. // NOTE: Ports and ServicePort cannot both be defined - ServicePort uint `hcl:"service_port,optional"` + ServicePort *uint `hcl:"service_port,optional"` // Environment variables that are meant to configure the application in a static - // way. This might be control an image that has mulitple modes of operation, + // way. This might be control an image that has multiple modes of operation, // selected via environment variable. Most configuration should use the waypoint // config commands. StaticEnvVars map[string]string `hcl:"static_environment,optional"` // Pod describes the configuration for the pod Pod *Pod `hcl:"pod,block"` + + // Deprecated field, previous definition of ports + DeprecatedPorts []map[string]string `hcl:"ports,optional" docs:"hidden"` } // ResourceConfig describes the request and limit of a resource. Used for @@ -1463,7 +1476,15 @@ type AutoscaleConfig struct { type Pod struct { SecurityContext *PodSecurityContext `hcl:"security_context,block"` Container *Container `hcl:"container,block"` - Sidecars []*Container `hcl:"sidecar,block"` + Sidecars []*Sidecar `hcl:"sidecar,block"` +} + +type Sidecar struct { + // Specifying Image in Container would make it visible on the main Pod config, + // which isn't the right way to specify the app image. + Image string `hcl:"image"` + + Container Container `hcl:"container,block"` } type Port struct { @@ -1476,8 +1497,7 @@ type Port struct { // Container describes the detailed parameters to declare a kubernetes container type Container struct { - Name string `hcl:"name"` - Image string `hcl:"image"` + Name string `hcl:"name,optional"` Ports []*Port `hcl:"port,block"` ProbePath string `hcl:"probe_path,optional"` Probe *Probe `hcl:"probe,block"` @@ -1529,213 +1549,286 @@ deploy "kubernetes" { } `) - doc.SetField( - "autoscale", - "sets up a horizontal pod autoscaler to scale deployments automatically", - docs.Summary("This configuration will automatically set up and associate the "+ - "current deployment with a horizontal pod autoscaler in Kuberentes. Note that "+ - "for this to work, you must also define resource limits and requests for a deployment "+ - "otherwise the metrics-server will not be able to properly determine a deployments "+ - "target CPU utilization"), - docs.SubFields(func(doc *docs.SubFieldDoc) { + setCommonVar := map[string]func(doc docs.DocField){ + "port": func(doc docs.DocField) { doc.SetField( - "min_replicas", - "The minimum amount of pods to have for a deployment", + "port", + "a port and options that the application is listening on", + docs.Summary( + "used to define and expose multiple ports that the application or process is", + "listening on for the container in use. Can be specified multiple times for many ports.", + ), + docs.SubFields(func(doc *docs.SubFieldDoc) { + doc.SetField( + "name", + "name of the port", + docs.Summary("If specified, this must be an IANA_SVC_NAME and unique within the pod. Each", + "named port in a pod must have a unique name. Name for the port that can be", + "referred to by services.", + ), + ) + doc.SetField( + "port", + "the port number", + docs.Summary("Number of port to expose on the pod's IP address.", + "This must be a valid port number, 0 < x < 65536.", + ), + ) + doc.SetField( + "host_port", + "the corresponding worker node port", + docs.Summary("Number of port to expose on the host.", + "If specified, this must be a valid port number, 0 < x < 65536.", + "If HostNetwork is specified, this must match ContainerPort.", + "Most containers do not need this.", + ), + ) + doc.SetField( + "host_ip", + "what host IP to bind the external port to", + ) + doc.SetField( + "protocol", + "protocol for port. Must be UDP, TCP, or SCTP", + docs.Default("TCP"), + ) + }), ) - + }, + "static_environment": func(doc docs.DocField) { doc.SetField( - "max_replicas", - "The maximum amount of pods to scale to for a deployment", + "static_environment", + "environment variables to control broad modes of the application", + docs.Summary( + "environment variables that are meant to configure the container in a static", + "way. This might be control an image that has multiple modes of operation,", + "selected via environment variable. Most configuration should use the waypoint", + "config commands", + ), ) - + }, + "cpu": func(doc docs.DocField) { doc.SetField( - "cpu_percent", - "The target CPU percent utilization before the horizontal pod autoscaler "+ - "scales up a deployments replicas", - ) - }), - ) + "cpu", + "cpu resource configuration", + docs.Summary("CPU lets you define resource limits and requests for a container in "+ + "a deployment."), + docs.SubFields(func(doc *docs.SubFieldDoc) { + doc.SetField( + "request", + "how much cpu to give the container in cpu cores. Supports m to indicate milli-cores", + ) - doc.SetField( - "pod", - "the configuration for a pod", - docs.Summary("Pod describes the configuration for a pod when deploying"), - docs.SubFields(func(doc *docs.SubFieldDoc) { + doc.SetField( + "limit", + "maximum amount of cpu to give the container. Supports m to indicate milli-cores", + ) + }), + ) + }, + "memory": func(doc docs.DocField) { doc.SetField( - "container", - "container describes the commands and arguments for a container config", + "memory", + "memory resource configuration", + docs.Summary("Memory lets you define resource limits and requests for a container in "+ + "a deployment."), docs.SubFields(func(doc *docs.SubFieldDoc) { doc.SetField( - "command", - "An array of strings to run for the container", + "request", + "how much memory to give the container in bytes. Supports k for kilobytes, m for megabytes, and g for gigabytes", ) doc.SetField( - "args", - "An array of string arguments to pass through to the container", + "limit", + "maximum amount of memory to give the container. Supports k for kilobytes, m for megabytes, and g for gigabytes", ) }), ) + }, + "resources": func(doc docs.DocField) { doc.SetField( - "pod_security_context", - "holds pod-level security attributes and container settings", + "resources", + "a map of resource limits and requests to apply to a pod on deploy", + docs.Summary( + "resource limits and requests for a container. This exists to allow any possible "+ + "resources. For cpu and memory, use those relevant settings instead. "+ + "Keys must start with either `limits_` or `requests_`. Any other options "+ + "will be ignored.", + ), + ) + }, + "probe_path": func(doc docs.DocField) { + doc.SetField( + "probe_path", + "the HTTP path to request to test that the application is running", + docs.Summary( + "without this, the test will simply be that the application has bound to the port", + ), + ) + }, + "probe": func(doc docs.DocField) { + doc.SetField( + "probe", + "configuration to control liveness and readiness probes", + docs.Summary("Probe describes a health check to be performed against a ", + "container to determine whether it is alive or ready to receive traffic."), docs.SubFields(func(doc *docs.SubFieldDoc) { doc.SetField( - "run_as_user", - "The UID to run the entrypoint of the container process", + "initial_delay", + "time in seconds to wait before performing the initial liveness and readiness probes", + docs.Default("5"), ) + doc.SetField( - "run_as_non_root", - "Indicates that the container must run as a non-root user", + "timeout", + "time in seconds before the probe fails", + docs.Default("5"), ) + doc.SetField( - "fs_group", - "A special supplemental group that applies to all containers in a pod", + "failure_threshold", + "number of times a liveness probe can fail before the container is killed", + docs.Summary( + "failureThreshold * TimeoutSeconds should be long enough to cover your worst case startup times", + ), + docs.Default("5"), ) + }), ) - }), - ) - - doc.SetField( - "kubeconfig", - "path to the kubeconfig file to use", - docs.Summary("by default uses from current user's home directory"), - docs.EnvVar("KUBECONFIG"), - ) - - doc.SetField( - "context", - "the kubectl context to use, as defined in the kubeconfig file", - ) + }, + } + setCommonVar["container"] = func(doc docs.DocField) { + doc.SetField( + "container", + "container describes the commands and arguments for a container config", + docs.SubFields(func(doc *docs.SubFieldDoc) { + doc.SetField( + "name", + "name of the container", + ) + + setCommonVar["cpu"](doc) + setCommonVar["memory"](doc) + setCommonVar["resources"](doc) + setCommonVar["probe_path"](doc) + setCommonVar["probe"](doc) + setCommonVar["port"](doc) + setCommonVar["static_environment"](doc) + + doc.SetField( + "command", + "an array of strings to run for the container", + ) + + doc.SetField( + "args", + "an array of string arguments to pass through to the container", + ) + }), + ) + } doc.SetField( - "replicas", - "the number of replicas to maintain", + "annotations", + "annotations to be added to the application pod", docs.Summary( - "if the replica count is maintained outside waypoint,", - "for instance by a pod autoscaler, do not set this variable", + "annotations are added to the pod spec of the deployed application. This is", + "useful when using mutating webhook admission controllers to further process", + "pod events.", ), ) doc.SetField( - "cpu", - "cpu resource configuration", - docs.Summary("CPU lets you define resource limits and requests for a pod in "+ - "a deployment."), + "autoscale", + "sets up a horizontal pod autoscaler to scale deployments automatically", + docs.Summary("This configuration will automatically set up and associate the "+ + "current deployment with a horizontal pod autoscaler in Kuberentes. Note that "+ + "for this to work, you must also define resource limits and requests for a deployment "+ + "otherwise the metrics-server will not be able to properly determine a deployments "+ + "target CPU utilization"), docs.SubFields(func(doc *docs.SubFieldDoc) { doc.SetField( - "request", - "how much cpu to give the pod in cpu cores. Supports m to inidicate milli-cores", - ) - - doc.SetField( - "limit", - "maximum amount of cpu to give the pod. Supports m to inidicate milli-cores", + "min_replicas", + "the minimum amount of pods to have for a deployment", ) - }), - ) - doc.SetField( - "memory", - "memory resource configuration", - docs.Summary("Memory lets you define resource limits and requests for a pod in "+ - "a deployment."), - docs.SubFields(func(doc *docs.SubFieldDoc) { doc.SetField( - "request", - "how much memory to give the pod in bytes. Supports k for kilobytes, m for megabytes, and g for gigabytes", + "max_replicas", + "the maximum amount of pods to scale to for a deployment", ) doc.SetField( - "limit", - "maximum amount of memory to give the pod. Supports k for kilobytes, m for megabytes, and g for gigabytes", + "cpu_percent", + "the target CPU percent utilization before the horizontal pod autoscaler "+ + "scales up a deployments replicas", ) }), ) doc.SetField( - "resources", - "a map of resource limits and requests to apply to a pod on deploy", - docs.Summary( - "resource limits and requests for a pod. This exists to allow any possible "+ - "resources. For cpu and memory, use those relevent settings instead. "+ - "Keys must start with either 'limits\\_' or 'requests\\_'. Any other options "+ - "will be ignored.", - ), + "context", + "the kubectl context to use, as defined in the kubeconfig file", ) doc.SetField( - "port", - "a port and options that the application is listening on", + "replicas", + "the number of replicas to maintain", docs.Summary( - "used to define and expose multiple ports that the application is", - "listening on for the container in use. Can be specified multiple times for many ports.", + "if the replica count is maintained outside waypoint,", + "for instance by a pod autoscaler, do not set this variable", ), ) doc.SetField( - "probe_path", - "the HTTP path to request to test that the application is running", + "image_secret", + "name of the Kubernetes secrete to use for the image", docs.Summary( - "without this, the test will simply be that the application has bound to the port", + "this references an existing secret, waypoint does not create this secret", ), ) doc.SetField( - "probe", - "configuration to control liveness and readiness probes", - docs.Summary("Probe describes a health check to be performed against a ", - "container to determine whether it is alive or ready to receive traffic."), - docs.SubFields(func(doc *docs.SubFieldDoc) { - doc.SetField( - "initial_delay", - "time in seconds to wait before performing the initial liveness and readiness probes", - docs.Default("5"), - ) - - doc.SetField( - "timeout", - "time in seconds before the probe fails", - docs.Default("5"), - ) - - doc.SetField( - "failure_threshold", - "number of times a liveness probe can fail before the container is killed", - docs.Summary( - "failureThreshold * TimeoutSeconds should be long enough to cover your worst case startup times", - ), - docs.Default("5"), - ) + "kubeconfig", + "path to the kubeconfig file to use", + docs.Summary("by default uses from current user's home directory"), + docs.EnvVar("KUBECONFIG"), + ) - }), + doc.SetField( + "labels", + "a map of key value labels to apply to the deployment pod", ) doc.SetField( - "scratch_path", - "a path for the service to store temporary data", + "namespace", + "namespace to target deployment into", docs.Summary( - "a path to a directory that will be created for the service to store temporary data using tmpfs", + "namespace is the name of the Kubernetes namespace to apply the deployment in.", + "This is useful to create deployments in non-default namespaces without creating kubeconfig contexts for each", ), ) + setCommonVar["probe_path"](doc) + setCommonVar["probe"](doc) + setCommonVar["resources"](doc) + setCommonVar["cpu"](doc) + setCommonVar["memory"](doc) + doc.SetField( - "image_secret", - "name of the Kubernetes secrete to use for the image", + "scratch_path", + "a path for the service to store temporary data", docs.Summary( - "this references an existing secret, waypoint does not create this secret", + "a path to a directory that will be created for the service to store temporary data using tmpfs", ), ) doc.SetField( - "static_environment", - "environment variables to control broad modes of the application", + "service_account", + "service account name to be added to the application pod", docs.Summary( - "environment variables that are meant to configure the application in a static", - "way. This might be control an image that has multiple modes of operation,", - "selected via environment variable. Most configuration should use the waypoint", - "config commands", + "service account is the name of the Kubernetes service account to add to the pod.", + "This is useful to apply Kubernetes RBAC to the application.", ), ) @@ -1749,37 +1842,47 @@ deploy "kubernetes" { ), ) - doc.SetField( - "annotations", - "annotations to be added to the application pod", - docs.Summary( - "annotations are added to the pod spec of the deployed application. This is", - "useful when using mutating webhook admission controllers to further process", - "pod events.", - ), - ) + setCommonVar["static_environment"](doc) doc.SetField( - "service_account", - "service account name to be added to the application pod", - docs.Summary( - "service account is the name of the Kubernetes service account to add to the pod.", - "This is useful to apply Kubernetes RBAC to the application.", - ), - ) - - doc.SetField( - "labels", - "a map of key value labels to apply to the deployment pod", - ) - - doc.SetField( - "namespace", - "namespace to target deployment into", - docs.Summary( - "namespace is the name of the Kubernetes namespace to apply the deployment in.", - "This is useful to create deployments in non-default namespaces without creating kubeconfig contexts for each", - ), + "pod", + "the configuration for a pod", + docs.Summary("Pod describes the configuration for a pod when deploying"), + docs.SubFields(func(doc *docs.SubFieldDoc) { + setCommonVar["container"](doc) + doc.SetField( + "sidecar", + "a sidecar container within the same pod", + docs.Summary("Another container to run alongside the app container in the kubernetes pod.", + "Can be specified multiple times for multiple sidecars.", + ), + docs.SubFields(func(doc *docs.SubFieldDoc) { + doc.SetField( + "image", + "image of the sidecar container", + ) + setCommonVar["container"](doc) + }), + ) + doc.SetField( + "pod_security_context", + "holds pod-level security attributes and container settings", + docs.SubFields(func(doc *docs.SubFieldDoc) { + doc.SetField( + "run_as_user", + "the UID to run the entrypoint of the container process", + ) + doc.SetField( + "run_as_non_root", + "indicates that the container must run as a non-root user", + ) + doc.SetField( + "fs_group", + "a special supplemental group that applies to all containers in a pod", + ) + }), + ) + }), ) return doc, nil diff --git a/go.mod b/go.mod index c7671e2e817..2c63316db77 100644 --- a/go.mod +++ b/go.mod @@ -64,7 +64,7 @@ require ( github.com/hashicorp/vault/api v1.0.5-0.20200519221902-385fac77e20f github.com/hashicorp/vault/sdk v0.1.14-0.20201202172114-ee5ebeb30fef github.com/hashicorp/waypoint-hzn v0.0.0-20201008221232-97cd4d9120b9 - github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210927182713-b1ec2b947355 + github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20211012012751-f91df594a7cb github.com/imdario/mergo v0.3.11 github.com/improbable-eng/grpc-web v0.13.0 github.com/jinzhu/now v1.1.1 // indirect diff --git a/go.sum b/go.sum index 762698c4e0d..fc9a6d5611e 100644 --- a/go.sum +++ b/go.sum @@ -1095,8 +1095,8 @@ github.com/hashicorp/vault/sdk v0.1.14-0.20201202172114-ee5ebeb30fef h1:YKouRHFf github.com/hashicorp/vault/sdk v0.1.14-0.20201202172114-ee5ebeb30fef/go.mod h1:cAGI4nVnEfAyMeqt9oB+Mase8DNn3qA/LDNHURiwssY= github.com/hashicorp/waypoint-hzn v0.0.0-20201008221232-97cd4d9120b9 h1:i9hzlv2SpmaNcQ8ZLGn01fp2Gqyejj0juVs7rYDgecE= github.com/hashicorp/waypoint-hzn v0.0.0-20201008221232-97cd4d9120b9/go.mod h1:ObgQSWSX9rsNofh16kctm6XxLW2QW1Ay6/9ris6T6DU= -github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210927182713-b1ec2b947355 h1:+8gR1iMTAwC7QBxo9BzXx8LphhjeVS3pfC1lMa/c5pU= -github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20210927182713-b1ec2b947355/go.mod h1:pJSjBDgumtkMumNuLh4wDYiRk/acWo5tzyUEFCC4ugQ= +github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20211012012751-f91df594a7cb h1:VlmHIiLZTZZLNI960vF3xJG2giZBEvn/Mo+/mpjDFDg= +github.com/hashicorp/waypoint-plugin-sdk v0.0.0-20211012012751-f91df594a7cb/go.mod h1:pJSjBDgumtkMumNuLh4wDYiRk/acWo5tzyUEFCC4ugQ= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20190923154419-df201c70410d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= diff --git a/internal/cli/app_docs.go b/internal/cli/app_docs.go index 70d70fb514d..957bc2763b2 100644 --- a/internal/cli/app_docs.go +++ b/internal/cli/app_docs.go @@ -690,6 +690,13 @@ func (c *AppDocsCommand) Run(args []string) int { return 1 } + if os.Getenv("WP_REATTACH_PLUGINS") != "" { + // Currently, only waypoint runners have the logic necessary to reattach to an existing plugin. + c.ui.Output("WP_REATTACH_PLUGINS detected, but plugin debugging is not supported with this command.", terminal.StatusError) + // Exit immediately, as an IDE user is unlikely to notice this warning otherwise + return 1 + } + if c.flagBuiltin { return c.builtinDocs(args) } diff --git a/website/content/partials/components/platform-kubernetes.mdx b/website/content/partials/components/platform-kubernetes.mdx index d441ccfef1f..7713d303130 100644 --- a/website/content/partials/components/platform-kubernetes.mdx +++ b/website/content/partials/components/platform-kubernetes.mdx @@ -49,18 +49,18 @@ The minimum amount of pods to have for a deployment. Cpu resource configuration. -CPU lets you define resource limits and requests for a pod in a deployment. +CPU lets you define resource limits and requests for a container in a deployment. ##### cpu.limit -Maximum amount of cpu to give the pod. Supports m to inidicate milli-cores. +Maximum amount of cpu to give the container. Supports m to indicate milli-cores. - Type: **string** - **Optional** ##### cpu.request -How much cpu to give the pod in cpu cores. Supports m to inidicate milli-cores. +How much cpu to give the container in cpu cores. Supports m to indicate milli-cores. - Type: **string** - **Optional** @@ -69,18 +69,18 @@ How much cpu to give the pod in cpu cores. Supports m to inidicate milli-cores. Memory resource configuration. -Memory lets you define resource limits and requests for a pod in a deployment. +Memory lets you define resource limits and requests for a container in a deployment. ##### memory.limit -Maximum amount of memory to give the pod. Supports k for kilobytes, m for megabytes, and g for gigabytes. +Maximum amount of memory to give the container. Supports k for kilobytes, m for megabytes, and g for gigabytes. - Type: **string** - **Optional** ##### memory.request -How much memory to give the pod in bytes. Supports k for kilobytes, m for megabytes, and g for gigabytes. +How much memory to give the container in bytes. Supports k for kilobytes, m for megabytes, and g for gigabytes. - Type: **string** - **Optional** @@ -103,6 +103,108 @@ An array of string arguments to pass through to the container. An array of strings to run for the container. +###### pod.container.cpu (category) + +Cpu resource configuration. + +CPU lets you define resource limits and requests for a container in a deployment. + +####### pod.container.cpu.limit + +Maximum amount of cpu to give the container. Supports m to indicate milli-cores. + +####### pod.container.cpu.request + +How much cpu to give the container in cpu cores. Supports m to indicate milli-cores. + +###### pod.container.memory (category) + +Memory resource configuration. + +Memory lets you define resource limits and requests for a container in a deployment. + +####### pod.container.memory.limit + +Maximum amount of memory to give the container. Supports k for kilobytes, m for megabytes, and g for gigabytes. + +####### pod.container.memory.request + +How much memory to give the container in bytes. Supports k for kilobytes, m for megabytes, and g for gigabytes. + +###### pod.container.name + +Name of the container. + +###### pod.container.port (category) + +A port and options that the application is listening on. + +Used to define and expose multiple ports that the application or process is listening on for the container in use. Can be specified multiple times for many ports. + +####### pod.container.port.host_ip + +What host IP to bind the external port to. + +####### pod.container.port.host_port + +The corresponding worker node port. + +Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this. + +####### pod.container.port.name + +Name of the port. + +If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services. + +####### pod.container.port.port + +The port number. + +Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. + +####### pod.container.port.protocol + +Protocol for port. Must be UDP, TCP, or SCTP. + +###### pod.container.probe (category) + +Configuration to control liveness and readiness probes. + +Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic. + +####### pod.container.probe.failure_threshold + +Number of times a liveness probe can fail before the container is killed. + +FailureThreshold \* TimeoutSeconds should be long enough to cover your worst case startup times. + +####### pod.container.probe.initial_delay + +Time in seconds to wait before performing the initial liveness and readiness probes. + +####### pod.container.probe.timeout + +Time in seconds before the probe fails. + +###### pod.container.probe_path + +The HTTP path to request to test that the application is running. + +Without this, the test will simply be that the application has bound to the port. + +###### pod.container.resources + +A map of resource limits and requests to apply to a pod on deploy. + +Resource limits and requests for a container. This exists to allow any possible resources. For cpu and memory, use those relevant settings instead. Keys must start with either `limits_` or `requests_`. Any other options will be ignored. + +###### pod.container.static_environment + +Environment variables to control broad modes of the application. + +Environment variables that are meant to configure the container in a static way. This might be control an image that has multiple modes of operation, selected via environment variable. Most configuration should use the waypoint config commands. + ##### pod.pod_security_context (category) Holds pod-level security attributes and container settings. @@ -123,17 +225,129 @@ The UID to run the entrypoint of the container process. - Type: **k8s.PodSecurityContext** -##### pod.sidecar +##### pod.sidecar (category) -- Type: **list of k8s.Container** +A sidecar container within the same pod. + +Another container to run alongside the app container in the kubernetes pod. Can be specified multiple times for multiple sidecars. + +###### pod.sidecar.container (category) + +Container describes the commands and arguments for a container config. -#### port +####### pod.sidecar.container.args + +An array of string arguments to pass through to the container. + +####### pod.sidecar.container.command + +An array of strings to run for the container. + +####### pod.sidecar.container.cpu (category) + +Cpu resource configuration. + +CPU lets you define resource limits and requests for a container in a deployment. + +######## pod.sidecar.container.cpu.limit + +Maximum amount of cpu to give the container. Supports m to indicate milli-cores. + +######## pod.sidecar.container.cpu.request + +How much cpu to give the container in cpu cores. Supports m to indicate milli-cores. + +####### pod.sidecar.container.memory (category) + +Memory resource configuration. + +Memory lets you define resource limits and requests for a container in a deployment. + +######## pod.sidecar.container.memory.limit + +Maximum amount of memory to give the container. Supports k for kilobytes, m for megabytes, and g for gigabytes. + +######## pod.sidecar.container.memory.request + +How much memory to give the container in bytes. Supports k for kilobytes, m for megabytes, and g for gigabytes. + +####### pod.sidecar.container.name + +Name of the container. + +####### pod.sidecar.container.port (category) A port and options that the application is listening on. -Used to define and expose multiple ports that the application is listening on for the container in use. Can be specified multiple times for many ports. +Used to define and expose multiple ports that the application or process is listening on for the container in use. Can be specified multiple times for many ports. + +######## pod.sidecar.container.port.host_ip + +What host IP to bind the external port to. + +######## pod.sidecar.container.port.host_port + +The corresponding worker node port. + +Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this. + +######## pod.sidecar.container.port.name + +Name of the port. + +If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services. + +######## pod.sidecar.container.port.port + +The port number. + +Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. + +######## pod.sidecar.container.port.protocol + +Protocol for port. Must be UDP, TCP, or SCTP. + +####### pod.sidecar.container.probe (category) + +Configuration to control liveness and readiness probes. + +Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic. + +######## pod.sidecar.container.probe.failure_threshold + +Number of times a liveness probe can fail before the container is killed. + +FailureThreshold \* TimeoutSeconds should be long enough to cover your worst case startup times. + +######## pod.sidecar.container.probe.initial_delay + +Time in seconds to wait before performing the initial liveness and readiness probes. + +######## pod.sidecar.container.probe.timeout + +Time in seconds before the probe fails. + +####### pod.sidecar.container.probe_path + +The HTTP path to request to test that the application is running. + +Without this, the test will simply be that the application has bound to the port. + +####### pod.sidecar.container.resources + +A map of resource limits and requests to apply to a pod on deploy. + +Resource limits and requests for a container. This exists to allow any possible resources. For cpu and memory, use those relevant settings instead. Keys must start with either `limits_` or `requests_`. Any other options will be ignored. + +####### pod.sidecar.container.static_environment + +Environment variables to control broad modes of the application. + +Environment variables that are meant to configure the container in a static way. This might be control an image that has multiple modes of operation, selected via environment variable. Most configuration should use the waypoint config commands. + +###### pod.sidecar.image -- Type: **list of k8s.Port** +Image of the sidecar container. #### probe (category) @@ -243,7 +457,7 @@ If the replica count is maintained outside waypoint, for instance by a pod autos A map of resource limits and requests to apply to a pod on deploy. -Resource limits and requests for a pod. This exists to allow any possible resources. For cpu and memory, use those relevent settings instead. Keys must start with either 'limits\_' or 'requests\_'. Any other options will be ignored. +Resource limits and requests for a container. This exists to allow any possible resources. For cpu and memory, use those relevant settings instead. Keys must start with either `limits_` or `requests_`. Any other options will be ignored. - Type: **map of string to string** - **Optional** @@ -280,7 +494,7 @@ By default, this config variable is used for exposing a single port for the cont Environment variables to control broad modes of the application. -Environment variables that are meant to configure the application in a static way. This might be control an image that has multiple modes of operation, selected via environment variable. Most configuration should use the waypoint config commands. +Environment variables that are meant to configure the container in a static way. This might be control an image that has multiple modes of operation, selected via environment variable. Most configuration should use the waypoint config commands. - Type: **map of string to string** - **Optional** From 2422e15477fc26999c505082f75b678420957a08 Mon Sep 17 00:00:00 2001 From: Izaak Lauer <8404559+izaaklauer@users.noreply.github.com> Date: Mon, 11 Oct 2021 21:38:48 -0400 Subject: [PATCH 08/13] Changelog tweak --- .changelog/2428.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changelog/2428.txt b/.changelog/2428.txt index 24d78c42fa8..c018517510f 100644 --- a/.changelog/2428.txt +++ b/.changelog/2428.txt @@ -4,5 +4,5 @@ plugin/k8s: Allows advanced users to add sidecar containers to apps using the k8 ``` release-note:breaking-change -plugin/k8s: "Ports" config is now []*Port rather than map[string]string -``` \ No newline at end of file +plugin/k8s: Advanced `Ports` config has a new type and has moves out of the top-level config. +``` From 8711cb8dd6565ae56ee27d4ceeadd6e7d7f8e5f6 Mon Sep 17 00:00:00 2001 From: Izaak Lauer <8404559+izaaklauer@users.noreply.github.com> Date: Mon, 11 Oct 2021 21:40:35 -0400 Subject: [PATCH 09/13] Resources apply to containers. --- builtin/k8s/platform.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/k8s/platform.go b/builtin/k8s/platform.go index c116ea1f313..7ecc74ed1fa 100644 --- a/builtin/k8s/platform.go +++ b/builtin/k8s/platform.go @@ -1648,7 +1648,7 @@ deploy "kubernetes" { "resources": func(doc docs.DocField) { doc.SetField( "resources", - "a map of resource limits and requests to apply to a pod on deploy", + "a map of resource limits and requests to apply to a container on deploy", docs.Summary( "resource limits and requests for a container. This exists to allow any possible "+ "resources. For cpu and memory, use those relevant settings instead. "+ From 7db9a0f89ef9de9e2957941326c16d79563fed02 Mon Sep 17 00:00:00 2001 From: Izaak Lauer <8404559+izaaklauer@users.noreply.github.com> Date: Tue, 12 Oct 2021 08:31:20 -0400 Subject: [PATCH 10/13] Rerun website-mdx with header limit --- .../components/platform-kubernetes.mdx | 74 +++++++++---------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/website/content/partials/components/platform-kubernetes.mdx b/website/content/partials/components/platform-kubernetes.mdx index 7713d303130..0886bc1934e 100644 --- a/website/content/partials/components/platform-kubernetes.mdx +++ b/website/content/partials/components/platform-kubernetes.mdx @@ -109,11 +109,11 @@ Cpu resource configuration. CPU lets you define resource limits and requests for a container in a deployment. -####### pod.container.cpu.limit +###### pod.container.cpu.limit Maximum amount of cpu to give the container. Supports m to indicate milli-cores. -####### pod.container.cpu.request +###### pod.container.cpu.request How much cpu to give the container in cpu cores. Supports m to indicate milli-cores. @@ -123,11 +123,11 @@ Memory resource configuration. Memory lets you define resource limits and requests for a container in a deployment. -####### pod.container.memory.limit +###### pod.container.memory.limit Maximum amount of memory to give the container. Supports k for kilobytes, m for megabytes, and g for gigabytes. -####### pod.container.memory.request +###### pod.container.memory.request How much memory to give the container in bytes. Supports k for kilobytes, m for megabytes, and g for gigabytes. @@ -141,29 +141,29 @@ A port and options that the application is listening on. Used to define and expose multiple ports that the application or process is listening on for the container in use. Can be specified multiple times for many ports. -####### pod.container.port.host_ip +###### pod.container.port.host_ip What host IP to bind the external port to. -####### pod.container.port.host_port +###### pod.container.port.host_port The corresponding worker node port. Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this. -####### pod.container.port.name +###### pod.container.port.name Name of the port. If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services. -####### pod.container.port.port +###### pod.container.port.port The port number. Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. -####### pod.container.port.protocol +###### pod.container.port.protocol Protocol for port. Must be UDP, TCP, or SCTP. @@ -173,17 +173,17 @@ Configuration to control liveness and readiness probes. Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic. -####### pod.container.probe.failure_threshold +###### pod.container.probe.failure_threshold Number of times a liveness probe can fail before the container is killed. FailureThreshold \* TimeoutSeconds should be long enough to cover your worst case startup times. -####### pod.container.probe.initial_delay +###### pod.container.probe.initial_delay Time in seconds to wait before performing the initial liveness and readiness probes. -####### pod.container.probe.timeout +###### pod.container.probe.timeout Time in seconds before the probe fails. @@ -195,7 +195,7 @@ Without this, the test will simply be that the application has bound to the port ###### pod.container.resources -A map of resource limits and requests to apply to a pod on deploy. +A map of resource limits and requests to apply to a container on deploy. Resource limits and requests for a container. This exists to allow any possible resources. For cpu and memory, use those relevant settings instead. Keys must start with either `limits_` or `requests_`. Any other options will be ignored. @@ -235,111 +235,111 @@ Another container to run alongside the app container in the kubernetes pod. Can Container describes the commands and arguments for a container config. -####### pod.sidecar.container.args +###### pod.sidecar.container.args An array of string arguments to pass through to the container. -####### pod.sidecar.container.command +###### pod.sidecar.container.command An array of strings to run for the container. -####### pod.sidecar.container.cpu (category) +###### pod.sidecar.container.cpu (category) Cpu resource configuration. CPU lets you define resource limits and requests for a container in a deployment. -######## pod.sidecar.container.cpu.limit +###### pod.sidecar.container.cpu.limit Maximum amount of cpu to give the container. Supports m to indicate milli-cores. -######## pod.sidecar.container.cpu.request +###### pod.sidecar.container.cpu.request How much cpu to give the container in cpu cores. Supports m to indicate milli-cores. -####### pod.sidecar.container.memory (category) +###### pod.sidecar.container.memory (category) Memory resource configuration. Memory lets you define resource limits and requests for a container in a deployment. -######## pod.sidecar.container.memory.limit +###### pod.sidecar.container.memory.limit Maximum amount of memory to give the container. Supports k for kilobytes, m for megabytes, and g for gigabytes. -######## pod.sidecar.container.memory.request +###### pod.sidecar.container.memory.request How much memory to give the container in bytes. Supports k for kilobytes, m for megabytes, and g for gigabytes. -####### pod.sidecar.container.name +###### pod.sidecar.container.name Name of the container. -####### pod.sidecar.container.port (category) +###### pod.sidecar.container.port (category) A port and options that the application is listening on. Used to define and expose multiple ports that the application or process is listening on for the container in use. Can be specified multiple times for many ports. -######## pod.sidecar.container.port.host_ip +###### pod.sidecar.container.port.host_ip What host IP to bind the external port to. -######## pod.sidecar.container.port.host_port +###### pod.sidecar.container.port.host_port The corresponding worker node port. Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this. -######## pod.sidecar.container.port.name +###### pod.sidecar.container.port.name Name of the port. If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services. -######## pod.sidecar.container.port.port +###### pod.sidecar.container.port.port The port number. Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. -######## pod.sidecar.container.port.protocol +###### pod.sidecar.container.port.protocol Protocol for port. Must be UDP, TCP, or SCTP. -####### pod.sidecar.container.probe (category) +###### pod.sidecar.container.probe (category) Configuration to control liveness and readiness probes. Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic. -######## pod.sidecar.container.probe.failure_threshold +###### pod.sidecar.container.probe.failure_threshold Number of times a liveness probe can fail before the container is killed. FailureThreshold \* TimeoutSeconds should be long enough to cover your worst case startup times. -######## pod.sidecar.container.probe.initial_delay +###### pod.sidecar.container.probe.initial_delay Time in seconds to wait before performing the initial liveness and readiness probes. -######## pod.sidecar.container.probe.timeout +###### pod.sidecar.container.probe.timeout Time in seconds before the probe fails. -####### pod.sidecar.container.probe_path +###### pod.sidecar.container.probe_path The HTTP path to request to test that the application is running. Without this, the test will simply be that the application has bound to the port. -####### pod.sidecar.container.resources +###### pod.sidecar.container.resources -A map of resource limits and requests to apply to a pod on deploy. +A map of resource limits and requests to apply to a container on deploy. Resource limits and requests for a container. This exists to allow any possible resources. For cpu and memory, use those relevant settings instead. Keys must start with either `limits_` or `requests_`. Any other options will be ignored. -####### pod.sidecar.container.static_environment +###### pod.sidecar.container.static_environment Environment variables to control broad modes of the application. @@ -455,7 +455,7 @@ If the replica count is maintained outside waypoint, for instance by a pod autos #### resources -A map of resource limits and requests to apply to a pod on deploy. +A map of resource limits and requests to apply to a container on deploy. Resource limits and requests for a container. This exists to allow any possible resources. For cpu and memory, use those relevant settings instead. Keys must start with either `limits_` or `requests_`. Any other options will be ignored. From 25f50bf31431f7505e1077fbf6de22447dd0dd7d Mon Sep 17 00:00:00 2001 From: Izaak Lauer <8404559+izaaklauer@users.noreply.github.com> Date: Tue, 12 Oct 2021 09:04:03 -0400 Subject: [PATCH 11/13] Fix bug in processing ServicePort overlay --- builtin/k8s/platform.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/k8s/platform.go b/builtin/k8s/platform.go index 7ecc74ed1fa..0fa53572005 100644 --- a/builtin/k8s/platform.go +++ b/builtin/k8s/platform.go @@ -1370,7 +1370,7 @@ func overlayTopLevelProperties(config Config, container *Container) (*Container, } if config.ServicePort != nil { // We've already validated that ports is nil in ConfigSet - they cannot both be set at once. - container.Ports = []*Port{{Port: *config.ServicePort, Name: "http"}} + overlaidContainer.Ports = []*Port{{Port: *config.ServicePort, Name: "http"}} } return overlaidContainer, nil From 8dfcb43cacd264ab698a433a23cb97a227474611 Mon Sep 17 00:00:00 2001 From: Izaak Lauer <8404559+izaaklauer@users.noreply.github.com> Date: Tue, 12 Oct 2021 09:31:21 -0400 Subject: [PATCH 12/13] Pointerifying structs to match our convention --- builtin/k8s/platform.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builtin/k8s/platform.go b/builtin/k8s/platform.go index 0fa53572005..517932e7d99 100644 --- a/builtin/k8s/platform.go +++ b/builtin/k8s/platform.go @@ -679,7 +679,7 @@ func (p *Platform) resourceDeploymentCreate( } sidecarContainer, err := configureContainer( - &sidecarConfig.Container, + sidecarConfig.Container, sidecarConfig.Image, envVars, p.config.ScratchSpace, @@ -1484,7 +1484,7 @@ type Sidecar struct { // which isn't the right way to specify the app image. Image string `hcl:"image"` - Container Container `hcl:"container,block"` + Container *Container `hcl:"container,block"` } type Port struct { From e1a32fe3330288730a50b78f471aa097f7d2edf6 Mon Sep 17 00:00:00 2001 From: Shirley Xiaolin Xu Date: Tue, 12 Oct 2021 14:16:20 -0400 Subject: [PATCH 13/13] respond to feedback --- .changelog/2428.txt | 2 +- builtin/k8s/platform.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.changelog/2428.txt b/.changelog/2428.txt index c018517510f..ad83f6ea42d 100644 --- a/.changelog/2428.txt +++ b/.changelog/2428.txt @@ -1,5 +1,5 @@ ```release-note:feature -plugin/k8s: Allows advanced users to add sidecar containers to apps using the k8s plugin config. +plugin/k8s: Allows users to add sidecar containers to apps using the k8s plugin config. ``` ``` diff --git a/builtin/k8s/platform.go b/builtin/k8s/platform.go index 517932e7d99..4ba6a975db8 100644 --- a/builtin/k8s/platform.go +++ b/builtin/k8s/platform.go @@ -143,7 +143,7 @@ func (p *Platform) ConfigSet(config interface{}) error { ), )) if err != nil { - return status.Errorf(codes.InvalidArgument, fmt.Sprintf("Invalid kubernetes platform plugin config: %s", err.Error())) + return status.Errorf(codes.InvalidArgument, "Invalid kubernetes platform plugin config: %s", err.Error()) } } @@ -621,7 +621,7 @@ func (p *Platform) resourceDeploymentCreate( // App container must have some kind of port if len(appContainerSpec.Ports) == 0 { - ui.Output(fmt.Sprintf("No ports defined - defaulting to http on port %d", DefaultServicePort), terminal.WithWarningStyle()) + ui.Output("No ports defined - defaulting to http on port %d", DefaultServicePort, terminal.WithWarningStyle()) appContainerSpec.Ports = append(appContainerSpec.Ports, &Port{Port: DefaultServicePort, Name: "http"}) }