diff --git a/go.mod b/go.mod index 83df7d5786..fa618970f2 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/anchore/stereoscope v0.0.1 github.com/anchore/syft v0.100.0 github.com/defenseunicorns/pkg/helpers/v2 v2.0.1 - github.com/defenseunicorns/pkg/kubernetes v0.0.1 + github.com/defenseunicorns/pkg/kubernetes v0.2.0 github.com/defenseunicorns/pkg/oci v1.0.1 github.com/derailed/k9s v0.31.7 github.com/distribution/reference v0.5.0 @@ -62,6 +62,11 @@ require ( sigs.k8s.io/yaml v1.4.0 ) +require ( + github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect +) + require ( atomicgo.dev/cursor v0.2.0 // indirect atomicgo.dev/keyboard v0.2.9 // indirect @@ -222,7 +227,6 @@ require ( github.com/emicklei/proto v1.12.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/evanphx/json-patch v5.7.0+incompatible // indirect - github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect github.com/facebookincubator/nvdtools v0.1.5 // indirect github.com/fatih/camelcase v1.0.0 // indirect @@ -244,7 +248,6 @@ require ( github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect - github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/analysis v0.22.0 // indirect github.com/go-openapi/errors v0.21.0 // indirect @@ -278,7 +281,7 @@ require ( github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.6.0 + github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.4 // indirect github.com/gookit/color v1.5.4 // indirect diff --git a/go.sum b/go.sum index b4033a097a..68aec72c72 100644 --- a/go.sum +++ b/go.sum @@ -599,8 +599,8 @@ github.com/defenseunicorns/gojsonschema v0.0.0-20231116163348-e00f069122d6 h1:gw github.com/defenseunicorns/gojsonschema v0.0.0-20231116163348-e00f069122d6/go.mod h1:StKLYMmPj1R5yIs6CK49EkcW1TvUYuw5Vri+LRk7Dy8= github.com/defenseunicorns/pkg/helpers/v2 v2.0.1 h1:j08rz9vhyD9Bs+yKiyQMY2tSSejXRMxTqEObZ5M1Wbk= github.com/defenseunicorns/pkg/helpers/v2 v2.0.1/go.mod h1:u1PAqOICZyiGIVA2v28g55bQH1GiAt0Bc4U9/rnWQvQ= -github.com/defenseunicorns/pkg/kubernetes v0.0.1 h1:HNQBV6XXFvlDvFdOCCWam0/LCgq67M+ggQKiRIoM2vU= -github.com/defenseunicorns/pkg/kubernetes v0.0.1/go.mod h1:AWB1iBbDO4VTmRO/E/8e0tVN0kkWbg+v8dhs9Hd9KXA= +github.com/defenseunicorns/pkg/kubernetes v0.2.0 h1:mVYZxmvpVa9zTC5U8wyNg1yoVAVkInJAg5qi9D0QoAQ= +github.com/defenseunicorns/pkg/kubernetes v0.2.0/go.mod h1:AWB1iBbDO4VTmRO/E/8e0tVN0kkWbg+v8dhs9Hd9KXA= github.com/defenseunicorns/pkg/oci v1.0.1 h1:WPrWRrae1L19X1vuhy6yYMR2zrTzgBbJHp3ImgUm4ZM= github.com/defenseunicorns/pkg/oci v1.0.1/go.mod h1:qZ3up/d0P81taW37fKR4lb19jJhQZJVtNOEJMu00dHQ= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da h1:ZOjWpVsFZ06eIhnh4mkaceTiVoktdU67+M7KDHJ268M= diff --git a/src/pkg/cluster/injector.go b/src/pkg/cluster/injector.go index 10b2778d7a..52824843bf 100644 --- a/src/pkg/cluster/injector.go +++ b/src/pkg/cluster/injector.go @@ -10,180 +10,137 @@ import ( "os" "path/filepath" "regexp" + "strings" "time" "github.com/google/go-containerregistry/pkg/crane" - "github.com/google/uuid" "github.com/mholt/archiver/v3" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" - "sigs.k8s.io/cli-utils/pkg/object" + "k8s.io/apimachinery/pkg/util/wait" "github.com/defenseunicorns/pkg/helpers/v2" pkgkubernetes "github.com/defenseunicorns/pkg/kubernetes" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" ) -// The chunk size for the tarball chunks. -var payloadChunkSize = 1024 * 768 - -var ( - injectorRequestedCPU = resource.MustParse(".5") - injectorRequestedMemory = resource.MustParse("64Mi") - injectorLimitCPU = resource.MustParse("1") - injectorLimitMemory = resource.MustParse("256Mi") -) - -// imageNodeMap is a map of image/node pairs. -type imageNodeMap map[string][]string +// StartInjection initializes a Zarf injection into the cluster. +func (c *Cluster) StartInjection(ctx context.Context, tmpDir, imagesDir string, injectorSeedSrcs []string) error { + // Stop any previous running injection before starting. + err := c.StopInjection(ctx) + if err != nil { + return err + } -// StartInjectionMadness initializes a Zarf injection into the cluster. -func (c *Cluster) StartInjectionMadness(ctx context.Context, tmpDir string, imagesDir string, injectorSeedSrcs []string) error { spinner := message.NewProgressSpinner("Attempting to bootstrap the seed image into the cluster") defer spinner.Stop() - tmp := layout.InjectionMadnessPaths{ - SeedImagesDir: filepath.Join(tmpDir, "seed-images"), - // should already exist - InjectionBinary: filepath.Join(tmpDir, "zarf-injector"), - // gets created here - InjectorPayloadTarGz: filepath.Join(tmpDir, "payload.tar.gz"), - } - - if err := helpers.CreateDirectory(tmp.SeedImagesDir, helpers.ReadWriteExecuteUser); err != nil { - return fmt.Errorf("unable to create the seed images directory: %w", err) + resReq := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(".5"), + corev1.ResourceMemory: resource.MustParse("64Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, } - - var err error - var images imageNodeMap - var payloadConfigmaps []string - var sha256sum string - - findImagesCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) - defer cancel() - images, err = c.getImagesAndNodesForInjection(findImagesCtx) + injectorImage, injectorNodeName, err := c.getInjectorImageAndNode(ctx, resReq) if err != nil { return err } - if err = c.createInjectorConfigMap(ctx, tmp.InjectionBinary); err != nil { - return fmt.Errorf("unable to create the injector configmap: %w", err) - } - - service, err := c.createService(ctx) - if err != nil { - return fmt.Errorf("unable to create the injector service: %w", err) - } - config.ZarfSeedPort = fmt.Sprintf("%d", service.Spec.Ports[0].NodePort) - - _, err = c.loadSeedImages(imagesDir, tmp.SeedImagesDir, injectorSeedSrcs) + payloadCmNames, shasum, err := c.createPayloadConfigMaps(ctx, spinner, tmpDir, imagesDir, injectorSeedSrcs) if err != nil { - return fmt.Errorf("unable to load the injector seed image from the package: %w", err) - } - - if payloadConfigmaps, sha256sum, err = c.createPayloadConfigMaps(ctx, tmp.SeedImagesDir, tmp.InjectorPayloadTarGz, spinner); err != nil { return fmt.Errorf("unable to generate the injector payload configmaps: %w", err) } - // https://regex101.com/r/eLS3at/1 - zarfImageRegex := regexp.MustCompile(`(?m)^127\.0\.0\.1:`) - - var injectorImage string - var injectorNode string - // Try to create an injector pod using an existing image in the cluster - for image, node := range images { - // Don't try to run against the seed image if this is a secondary zarf init run - if zarfImageRegex.MatchString(image) { - continue - } - spinner.Updatef("Attempting to bootstrap with the %s/%s", node, image) - injectorImage = image - injectorNode = node[0] - } - // Make sure the pod is not there first - // TODO: Explain why no grace period is given. - deleteGracePeriod := int64(0) - deletePolicy := metav1.DeletePropagationForeground - deleteOpts := metav1.DeleteOptions{ - GracePeriodSeconds: &deleteGracePeriod, - PropagationPolicy: &deletePolicy, - } - selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "zarf-injector", - }, - }) + b, err := os.ReadFile(filepath.Join(tmpDir, "zarf-injector")) if err != nil { return err } - listOpts := metav1.ListOptions{ - LabelSelector: selector.String(), + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ZarfNamespaceName, + Name: "rust-binary", + }, + BinaryData: map[string][]byte{ + "zarf-injector": b, + }, } - err = c.Clientset.CoreV1().Pods(ZarfNamespaceName).DeleteCollection(ctx, deleteOpts, listOpts) + _, err = c.Clientset.CoreV1().ConfigMaps(cm.Namespace).Create(ctx, cm, metav1.CreateOptions{}) if err != nil { return err } - pod, err := c.buildInjectionPod(injectorNode, injectorImage, payloadConfigmaps, sha256sum) + svc := &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Service", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ZarfNamespaceName, + Name: "zarf-injector", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + Ports: []corev1.ServicePort{ + { + Port: int32(5000), + }, + }, + Selector: map[string]string{ + "app": "zarf-injector", + }, + }, + } + svc, err = c.Clientset.CoreV1().Services(svc.Namespace).Create(ctx, svc, metav1.CreateOptions{}) if err != nil { - return fmt.Errorf("error making injection pod: %w", err) + return err } + // TODO: Remove use of passing data through global variables. + config.ZarfSeedPort = fmt.Sprintf("%d", svc.Spec.Ports[0].NodePort) - pod, err = c.Clientset.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + pod := buildInjectionPod(injectorNodeName, injectorImage, payloadCmNames, shasum, resReq) + _, err = c.Clientset.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("error creating pod in cluster: %w", err) } - objs := []object.ObjMetadata{ - { - GroupKind: schema.GroupKind{ - Kind: "Pod", - }, - Namespace: ZarfNamespaceName, - Name: pod.Name, - }, - } waitCtx, waitCancel := context.WithTimeout(ctx, 60*time.Second) defer waitCancel() - err = pkgkubernetes.WaitForReady(waitCtx, c.Watcher, objs) + err = pkgkubernetes.WaitForReadyRuntime(waitCtx, c.Watcher, []runtime.Object{pod}) if err != nil { return err } + spinner.Success() return nil - } -// StopInjectionMadness handles cleanup once the seed registry is up. -func (c *Cluster) StopInjectionMadness(ctx context.Context) error { - // Try to kill the injector pod now - selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "zarf-injector", - }, - }) - if err != nil { +// StopInjection handles cleanup once the seed registry is up. +func (c *Cluster) StopInjection(ctx context.Context) error { + err := c.Clientset.CoreV1().Pods(ZarfNamespaceName).Delete(ctx, "injector", metav1.DeleteOptions{}) + if err != nil && !kerrors.IsNotFound(err) { return err } - listOpts := metav1.ListOptions{ - LabelSelector: selector.String(), + err = c.Clientset.CoreV1().Services(ZarfNamespaceName).Delete(ctx, "zarf-injector", metav1.DeleteOptions{}) + if err != nil && !kerrors.IsNotFound(err) { + return err } - err = c.Clientset.CoreV1().Pods(ZarfNamespaceName).DeleteCollection(ctx, metav1.DeleteOptions{}, listOpts) - if err != nil { + err = c.Clientset.CoreV1().ConfigMaps(ZarfNamespaceName).Delete(ctx, "rust-binary", metav1.DeleteOptions{}) + if err != nil && !kerrors.IsNotFound(err) { return err } - - // Remove the configmaps - selector, err = metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ + selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ MatchLabels: map[string]string{ "zarf-injector": "payload", }, @@ -191,7 +148,7 @@ func (c *Cluster) StopInjectionMadness(ctx context.Context) error { if err != nil { return err } - listOpts = metav1.ListOptions{ + listOpts := metav1.ListOptions{ LabelSelector: selector.String(), } err = c.Clientset.CoreV1().ConfigMaps(ZarfNamespaceName).DeleteCollection(ctx, metav1.DeleteOptions{}, listOpts) @@ -199,89 +156,93 @@ func (c *Cluster) StopInjectionMadness(ctx context.Context) error { return err } - // Remove the injector service - err = c.Clientset.CoreV1().Services(ZarfNamespaceName).Delete(ctx, "zarf-injector", metav1.DeleteOptions{}) + // This is needed because labels were not present in payload config maps previously. + // Without this injector will fail if the config maps exist from a previous Zarf version. + cmList, err := c.Clientset.CoreV1().ConfigMaps(ZarfNamespaceName).List(ctx, metav1.ListOptions{}) + if err != nil { + return err + } + for _, cm := range cmList.Items { + if !strings.HasPrefix(cm.Name, "zarf-payload-") { + continue + } + err = c.Clientset.CoreV1().ConfigMaps(ZarfNamespaceName).Delete(ctx, cm.Name, metav1.DeleteOptions{}) + if err != nil { + return err + } + } + + // TODO: Replace with wait package in the future. + err = wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { + _, err := c.Clientset.CoreV1().Pods(ZarfNamespaceName).Get(ctx, "injector", metav1.GetOptions{}) + if kerrors.IsNotFound(err) { + return true, nil + } + return false, err + }) if err != nil { return err } return nil } -func (c *Cluster) loadSeedImages(imagesDir, seedImagesDir string, injectorSeedSrcs []string) ([]transform.Image, error) { - seedImages := []transform.Image{} - localReferenceToDigest := make(map[string]string) +func (c *Cluster) createPayloadConfigMaps(ctx context.Context, spinner *message.Spinner, tmpDir, imagesDir string, injectorSeedSrcs []string) ([]string, string, error) { + tarPath := filepath.Join(tmpDir, "payload.tar.gz") + seedImagesDir := filepath.Join(tmpDir, "seed-images") + if err := helpers.CreateDirectory(seedImagesDir, helpers.ReadWriteExecuteUser); err != nil { + return nil, "", fmt.Errorf("unable to create the seed images directory: %w", err) + } - // Load the injector-specific images and save them as seed-images + localReferenceToDigest := map[string]string{} for _, src := range injectorSeedSrcs { ref, err := transform.ParseImageRef(src) if err != nil { - return nil, fmt.Errorf("failed to create ref for image %s: %w", src, err) + return nil, "", fmt.Errorf("failed to create ref for image %s: %w", src, err) } img, err := utils.LoadOCIImage(imagesDir, ref) if err != nil { - return nil, err + return nil, "", err } - if err := crane.SaveOCI(img, seedImagesDir); err != nil { - return nil, err + return nil, "", err } - - seedImages = append(seedImages, ref) - - // Get the image digest so we can set an annotation in the image.json later imgDigest, err := img.Digest() if err != nil { - return nil, err + return nil, "", err } - // This is done _without_ the domain (different from pull.go) since the injector only handles local images localReferenceToDigest[ref.Path+ref.TagOrDigest] = imgDigest.String() } - if err := utils.AddImageNameAnnotation(seedImagesDir, localReferenceToDigest); err != nil { - return nil, fmt.Errorf("unable to format OCI layout: %w", err) + return nil, "", fmt.Errorf("unable to format OCI layout: %w", err) } - return seedImages, nil -} - -func (c *Cluster) createPayloadConfigMaps(ctx context.Context, seedImagesDir, tarPath string, spinner *message.Spinner) ([]string, string, error) { - var configMaps []string - // Chunk size has to accommodate base64 encoding & etcd 1MB limit tarFileList, err := filepath.Glob(filepath.Join(seedImagesDir, "*")) if err != nil { - return configMaps, "", err + return nil, "", err } - - // Create a tar archive of the injector payload if err := archiver.Archive(tarFileList, tarPath); err != nil { - return configMaps, "", err + return nil, "", err } - - chunks, sha256sum, err := helpers.ReadFileByChunks(tarPath, payloadChunkSize) + payloadChunkSize := 1024 * 768 + chunks, shasum, err := helpers.ReadFileByChunks(tarPath, payloadChunkSize) if err != nil { - return configMaps, "", err + return nil, "", err } - chunkCount := len(chunks) + cmNames := []string{} + for i, data := range chunks { + fileName := fmt.Sprintf("zarf-payload-%03d", i) - // Loop over all chunks and generate configmaps - for idx, data := range chunks { - // Create a cat-friendly filename - fileName := fmt.Sprintf("zarf-payload-%03d", idx) + spinner.Updatef("Adding archive binary configmap %d of %d to the cluster", i+1, len(chunks)) - spinner.Updatef("Adding archive binary configmap %d of %d to the cluster", idx+1, chunkCount) - - // Attempt to create the configmap in the cluster - // TODO: Replace with create or update. - err := c.Clientset.CoreV1().ConfigMaps(ZarfNamespaceName).Delete(ctx, fileName, metav1.DeleteOptions{}) - if err != nil && !kerrors.IsNotFound(err) { - return nil, "", err - } cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: fileName, Namespace: ZarfNamespaceName, + Name: fileName, + Labels: map[string]string{ + "zarf-injector": "payload", + }, }, BinaryData: map[string][]byte{ fileName: data, @@ -291,94 +252,81 @@ func (c *Cluster) createPayloadConfigMaps(ctx context.Context, seedImagesDir, ta if err != nil { return nil, "", err } - - // Add the configmap to the configmaps slice for later usage in the pod - configMaps = append(configMaps, fileName) + cmNames = append(cmNames, fileName) // Give the control plane a 250ms buffer between each configmap time.Sleep(250 * time.Millisecond) } - - return configMaps, sha256sum, nil + return cmNames, shasum, nil } -func (c *Cluster) createInjectorConfigMap(ctx context.Context, binaryPath string) error { - name := "rust-binary" - // TODO: Replace with a create or update. - err := c.Clientset.CoreV1().ConfigMaps(ZarfNamespaceName).Delete(ctx, name, metav1.DeleteOptions{}) - if err != nil && !kerrors.IsNotFound(err) { - return err - } - b, err := os.ReadFile(binaryPath) +// getImagesAndNodesForInjection checks for images on schedulable nodes within a cluster. +func (c *Cluster) getInjectorImageAndNode(ctx context.Context, resReq corev1.ResourceRequirements) (string, string, error) { + // Regex for Zarf seed image + zarfImageRegex, err := regexp.Compile(`(?m)^127\.0\.0\.1:`) if err != nil { - return err + return "", "", err } - configData := map[string][]byte{ - "zarf-injector": b, - } - configMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: ZarfNamespaceName, - }, - BinaryData: configData, + listOpts := metav1.ListOptions{ + FieldSelector: fmt.Sprintf("status.phase=%s", corev1.PodRunning), } - _, err = c.Clientset.CoreV1().ConfigMaps(configMap.Namespace).Create(ctx, configMap, metav1.CreateOptions{}) + podList, err := c.Clientset.CoreV1().Pods(corev1.NamespaceAll).List(ctx, listOpts) if err != nil { - return err + return "", "", err } - return nil + for _, pod := range podList.Items { + nodeDetails, err := c.Clientset.CoreV1().Nodes().Get(ctx, pod.Spec.NodeName, metav1.GetOptions{}) + if err != nil { + return "", "", err + } + if nodeDetails.Status.Allocatable.Cpu().Cmp(resReq.Requests[corev1.ResourceCPU]) < 0 || + nodeDetails.Status.Allocatable.Memory().Cmp(resReq.Requests[corev1.ResourceMemory]) < 0 { + continue + } + if hasBlockingTaints(nodeDetails.Spec.Taints) { + continue + } + for _, container := range pod.Spec.Containers { + if zarfImageRegex.MatchString(container.Image) { + continue + } + return container.Image, pod.Spec.NodeName, nil + } + for _, container := range pod.Spec.InitContainers { + if zarfImageRegex.MatchString(container.Image) { + continue + } + return container.Image, pod.Spec.NodeName, nil + } + for _, container := range pod.Spec.EphemeralContainers { + if zarfImageRegex.MatchString(container.Image) { + continue + } + return container.Image, pod.Spec.NodeName, nil + } + } + return "", "", fmt.Errorf("no suitable injector image or node exists") } -func (c *Cluster) createService(ctx context.Context) (*corev1.Service, error) { - svc := &corev1.Service{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "Service", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "zarf-injector", - Namespace: ZarfNamespaceName, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeNodePort, - Ports: []corev1.ServicePort{ - { - Port: int32(5000), - }, - }, - Selector: map[string]string{ - "app": "zarf-injector", - }, - }, - } - // TODO: Replace with create or update - err := c.Clientset.CoreV1().Services(svc.Namespace).Delete(ctx, svc.Name, metav1.DeleteOptions{}) - if err != nil && !kerrors.IsNotFound(err) { - return nil, err - } - svc, err = c.Clientset.CoreV1().Services(svc.Namespace).Create(ctx, svc, metav1.CreateOptions{}) - if err != nil { - return nil, err +func hasBlockingTaints(taints []corev1.Taint) bool { + for _, taint := range taints { + if taint.Effect == corev1.TaintEffectNoSchedule || taint.Effect == corev1.TaintEffectNoExecute { + return true + } } - return svc, nil + return false } -// buildInjectionPod return a pod for injection with the appropriate containers to perform the injection. -func (c *Cluster) buildInjectionPod(node, image string, payloadConfigmaps []string, payloadShasum string) (*corev1.Pod, error) { +func buildInjectionPod(nodeName, image string, payloadCmNames []string, shasum string, resReq corev1.ResourceRequirements) *corev1.Pod { executeMode := int32(0777) - // Generate a UUID to append to the pod name. - // This prevents collisions where `zarf init` is ran back to back and a previous injector pod still exists. - uuid := uuid.New().String()[:16] - pod := &corev1.Pod{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "Pod", }, ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("injector-%s", uuid), + Name: "injector", Namespace: ZarfNamespaceName, Labels: map[string]string{ "app": "zarf-injector", @@ -386,26 +334,16 @@ func (c *Cluster) buildInjectionPod(node, image string, payloadConfigmaps []stri }, }, Spec: corev1.PodSpec{ - NodeName: node, - // Do not try to restart the pod as it will be deleted/re-created instead + NodeName: nodeName, + // Do not try to restart the pod as it will be deleted/re-created instead. RestartPolicy: corev1.RestartPolicyNever, Containers: []corev1.Container{ { - Name: "injector", - - // An existing image already present on the cluster - Image: image, - - // PullIfNotPresent because some distros provide a way (even in airgap) to pull images from local or direct-connected registries + Name: "injector", + Image: image, ImagePullPolicy: corev1.PullIfNotPresent, - - // This directory's contents come from the init container output - WorkingDir: "/zarf-init", - - // Call the injector with shasum of the tarball - Command: []string{"/zarf-init/zarf-injector", payloadShasum}, - - // Shared mount between the init and regular containers + WorkingDir: "/zarf-init", + Command: []string{"/zarf-init/zarf-injector", shasum}, VolumeMounts: []corev1.VolumeMount{ { Name: "init", @@ -417,31 +355,18 @@ func (c *Cluster) buildInjectionPod(node, image string, payloadConfigmaps []stri MountPath: "/zarf-seed", }, }, - - // Readiness probe to optimize the pod startup time ReadinessProbe: &corev1.Probe{ PeriodSeconds: 2, SuccessThreshold: 1, FailureThreshold: 10, ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: "/v2/", // path to health check - Port: intstr.FromInt(5000), // port to health check + Path: "/v2/", + Port: intstr.FromInt(5000), }, }, }, - - // Keep resources as light as possible as we aren't actually running the container's other binaries - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: injectorRequestedCPU, - corev1.ResourceMemory: injectorRequestedMemory, - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: injectorLimitCPU, - corev1.ResourceMemory: injectorLimitMemory, - }, - }, + Resources: resReq, }, }, Volumes: []corev1.Volume{ @@ -468,9 +393,7 @@ func (c *Cluster) buildInjectionPod(node, image string, payloadConfigmaps []stri }, } - // Iterate over all the payload configmaps and add their mounts. - for _, filename := range payloadConfigmaps { - // Create the configmap volume from the given filename. + for _, filename := range payloadCmNames { pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ Name: filename, VolumeSource: corev1.VolumeSource{ @@ -481,8 +404,6 @@ func (c *Cluster) buildInjectionPod(node, image string, payloadConfigmaps []stri }, }, }) - - // Create the volume mount to place the new volume in the working directory pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ Name: filename, MountPath: fmt.Sprintf("/zarf-init/%s", filename), @@ -490,72 +411,5 @@ func (c *Cluster) buildInjectionPod(node, image string, payloadConfigmaps []stri }) } - return pod, nil -} - -// getImagesAndNodesForInjection checks for images on schedulable nodes within a cluster. -func (c *Cluster) getImagesAndNodesForInjection(ctx context.Context) (imageNodeMap, error) { - result := make(imageNodeMap) - - timer := time.NewTimer(0) - defer timer.Stop() - - for { - select { - case <-ctx.Done(): - return nil, fmt.Errorf("get image list timed-out: %w", ctx.Err()) - case <-timer.C: - listOpts := metav1.ListOptions{ - FieldSelector: fmt.Sprintf("status.phase=%s", corev1.PodRunning), - } - podList, err := c.Clientset.CoreV1().Pods(corev1.NamespaceAll).List(ctx, listOpts) - if err != nil { - return nil, fmt.Errorf("unable to get the list of %q pods in the cluster: %w", corev1.PodRunning, err) - } - - for _, pod := range podList.Items { - nodeName := pod.Spec.NodeName - - nodeDetails, err := c.Clientset.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) - if err != nil { - return nil, fmt.Errorf("unable to get the node %q: %w", nodeName, err) - } - - if nodeDetails.Status.Allocatable.Cpu().Cmp(injectorRequestedCPU) < 0 || - nodeDetails.Status.Allocatable.Memory().Cmp(injectorRequestedMemory) < 0 { - continue - } - - if hasBlockingTaints(nodeDetails.Spec.Taints) { - continue - } - - for _, container := range pod.Spec.InitContainers { - result[container.Image] = append(result[container.Image], nodeName) - } - for _, container := range pod.Spec.Containers { - result[container.Image] = append(result[container.Image], nodeName) - } - for _, container := range pod.Spec.EphemeralContainers { - result[container.Image] = append(result[container.Image], nodeName) - } - } - - if len(result) > 0 { - return result, nil - } - - message.Debug("No images found on any node. Retrying...") - timer.Reset(2 * time.Second) - } - } -} - -func hasBlockingTaints(taints []corev1.Taint) bool { - for _, taint := range taints { - if taint.Effect == corev1.TaintEffectNoSchedule || taint.Effect == corev1.TaintEffectNoExecute { - return true - } - } - return false + return pod } diff --git a/src/pkg/cluster/injector_test.go b/src/pkg/cluster/injector_test.go index 291eb092ef..07abc3bb6c 100644 --- a/src/pkg/cluster/injector_test.go +++ b/src/pkg/cluster/injector_test.go @@ -11,69 +11,152 @@ import ( "path/filepath" "strings" "testing" - "time" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/random" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" -) - -func TestCreateInjectorConfigMap(t *testing.T) { - t.Parallel() + k8stesting "k8s.io/client-go/testing" + "sigs.k8s.io/cli-utils/pkg/kstatus/status" - binData := []byte("foobar") - binPath := filepath.Join(t.TempDir(), "bin") - err := os.WriteFile(binPath, binData, 0o644) - require.NoError(t, err) + pkgkubernetes "github.com/defenseunicorns/pkg/kubernetes" +) +func TestInjector(t *testing.T) { + ctx := context.Background() cs := fake.NewSimpleClientset() c := &Cluster{ Clientset: cs, + Watcher: pkgkubernetes.NewImmediateWatcher(status.CurrentStatus), } + cs.PrependReactor("delete-collection", "configmaps", func(action k8stesting.Action) (bool, runtime.Object, error) { + delAction := action.(k8stesting.DeleteCollectionActionImpl) + if delAction.GetListRestrictions().Labels.String() != "zarf-injector=payload" { + return false, nil, nil + } + gvr := delAction.Resource + gvk := delAction.Resource.GroupVersion().WithKind("ConfigMap") + list, err := cs.Tracker().List(gvr, gvk, delAction.Namespace) + require.NoError(t, err) + for _, cm := range list.(*corev1.ConfigMapList).Items { + v, ok := cm.Labels["zarf-injector"] + if !ok { + continue + } + if v != "payload" { + continue + } + err = cs.Tracker().Delete(gvr, delAction.Namespace, cm.Name) + require.NoError(t, err) + } + return true, nil, nil + }) - ctx := context.Background() - for i := 0; i < 2; i++ { - err = c.createInjectorConfigMap(ctx, binPath) + // Setup nodes and pods with images + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Status: corev1.NodeStatus{ + Allocatable: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10"), + corev1.ResourceMemory: resource.MustParse("100Gi"), + }, + }, + } + _, err := cs.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) + require.NoError(t, err) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "good", + Namespace: "default", + }, + Spec: corev1.PodSpec{ + NodeName: "node1", + Containers: []corev1.Container{ + { + Image: "ubuntu:latest", + }, + }, + }, + } + _, err = cs.CoreV1().Pods(pod.ObjectMeta.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + require.NoError(t, err) + + err = c.StopInjection(ctx) + require.NoError(t, err) + + for range 2 { + tmpDir := t.TempDir() + binData := []byte("foobar") + err := os.WriteFile(filepath.Join(tmpDir, "zarf-injector"), binData, 0o644) require.NoError(t, err) - cm, err := cs.CoreV1().ConfigMaps(ZarfNamespaceName).Get(ctx, "rust-binary", metav1.GetOptions{}) + + idx, err := random.Index(1, 1, 1) + require.NoError(t, err) + _, err = layout.Write(filepath.Join(tmpDir, "seed-images"), idx) require.NoError(t, err) - require.Equal(t, binData, cm.BinaryData["zarf-injector"]) - } -} -func TestCreateService(t *testing.T) { - t.Parallel() + err = c.StartInjection(ctx, tmpDir, t.TempDir(), nil) + require.NoError(t, err) - cs := fake.NewSimpleClientset() - c := &Cluster{ - Clientset: cs, - } + podList, err := cs.CoreV1().Pods(ZarfNamespaceName).List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Len(t, podList.Items, 1) + require.Equal(t, "injector", podList.Items[0].ObjectMeta.Name) - expected, err := os.ReadFile("./testdata/expected-injection-service.json") - require.NoError(t, err) - ctx := context.Background() - for i := 0; i < 2; i++ { - _, err := c.createService(ctx) + svcList, err := cs.CoreV1().Services(ZarfNamespaceName).List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Len(t, svcList.Items, 1) + expected, err := os.ReadFile("./testdata/expected-injection-service.json") require.NoError(t, err) svc, err := cs.CoreV1().Services(ZarfNamespaceName).Get(ctx, "zarf-injector", metav1.GetOptions{}) require.NoError(t, err) b, err := json.Marshal(svc) require.NoError(t, err) require.Equal(t, strings.TrimSpace(string(expected)), string(b)) + + cmList, err := cs.CoreV1().ConfigMaps(ZarfNamespaceName).List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Len(t, cmList.Items, 2) + cm, err := cs.CoreV1().ConfigMaps(ZarfNamespaceName).Get(ctx, "rust-binary", metav1.GetOptions{}) + require.NoError(t, err) + require.Equal(t, binData, cm.BinaryData["zarf-injector"]) } + + err = c.StopInjection(ctx) + require.NoError(t, err) + + podList, err := cs.CoreV1().Pods(ZarfNamespaceName).List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Empty(t, podList.Items) + svcList, err := cs.CoreV1().Services(ZarfNamespaceName).List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Empty(t, svcList.Items) + cmList, err := cs.CoreV1().ConfigMaps(ZarfNamespaceName).List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Empty(t, cmList.Items) } func TestBuildInjectionPod(t *testing.T) { t.Parallel() - c := &Cluster{} - pod, err := c.buildInjectionPod("injection-node", "docker.io/library/ubuntu:latest", []string{"foo", "bar"}, "shasum") - require.NoError(t, err) - require.Contains(t, pod.Name, "injector-") - // Replace the random UUID in the pod name with a fixed placeholder for consistent comparison. - pod.ObjectMeta.Name = "injector-UUID" + resReq := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(".5"), + corev1.ResourceMemory: resource.MustParse("64Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + } + pod := buildInjectionPod("injection-node", "docker.io/library/ubuntu:latest", []string{"foo", "bar"}, "shasum", resReq) + require.Equal(t, "injector", pod.Name) b, err := json.Marshal(pod) require.NoError(t, err) expected, err := os.ReadFile("./testdata/expected-injection-pod.json") @@ -81,7 +164,7 @@ func TestBuildInjectionPod(t *testing.T) { require.Equal(t, strings.TrimSpace(string(expected)), string(b)) } -func TestImagesAndNodesForInjection(t *testing.T) { +func TestGetInjectorImageAndNode(t *testing.T) { t.Parallel() ctx := context.Background() @@ -185,14 +268,18 @@ func TestImagesAndNodesForInjection(t *testing.T) { require.NoError(t, err) } - getCtx, getCancel := context.WithTimeout(ctx, 1*time.Second) - defer getCancel() - result, err := c.getImagesAndNodesForInjection(getCtx) - require.NoError(t, err) - expected := imageNodeMap{ - "pod-2-init": []string{"good"}, - "pod-2-container": []string{"good"}, - "pod-2-ephemeral": []string{"good"}, + resReq := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(".5"), + corev1.ResourceMemory: resource.MustParse("64Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, } - require.Equal(t, expected, result) + image, node, err := c.getInjectorImageAndNode(ctx, resReq) + require.NoError(t, err) + require.Equal(t, "pod-2-container", image) + require.Equal(t, "good", node) } diff --git a/src/pkg/cluster/testdata/expected-injection-pod.json b/src/pkg/cluster/testdata/expected-injection-pod.json index a0c41072d3..30f2e5b1f1 100644 --- a/src/pkg/cluster/testdata/expected-injection-pod.json +++ b/src/pkg/cluster/testdata/expected-injection-pod.json @@ -1 +1 @@ -{"kind":"Pod","apiVersion":"v1","metadata":{"name":"injector-UUID","namespace":"zarf","creationTimestamp":null,"labels":{"app":"zarf-injector","zarf.dev/agent":"ignore"}},"spec":{"volumes":[{"name":"init","configMap":{"name":"rust-binary","defaultMode":511}},{"name":"seed","emptyDir":{}},{"name":"foo","configMap":{"name":"foo"}},{"name":"bar","configMap":{"name":"bar"}}],"containers":[{"name":"injector","image":"docker.io/library/ubuntu:latest","command":["/zarf-init/zarf-injector","shasum"],"workingDir":"/zarf-init","resources":{"limits":{"cpu":"1","memory":"256Mi"},"requests":{"cpu":"500m","memory":"64Mi"}},"volumeMounts":[{"name":"init","mountPath":"/zarf-init/zarf-injector","subPath":"zarf-injector"},{"name":"seed","mountPath":"/zarf-seed"},{"name":"foo","mountPath":"/zarf-init/foo","subPath":"foo"},{"name":"bar","mountPath":"/zarf-init/bar","subPath":"bar"}],"readinessProbe":{"httpGet":{"path":"/v2/","port":5000},"periodSeconds":2,"successThreshold":1,"failureThreshold":10},"imagePullPolicy":"IfNotPresent"}],"restartPolicy":"Never","nodeName":"injection-node"},"status":{}} +{"kind":"Pod","apiVersion":"v1","metadata":{"name":"injector","namespace":"zarf","creationTimestamp":null,"labels":{"app":"zarf-injector","zarf.dev/agent":"ignore"}},"spec":{"volumes":[{"name":"init","configMap":{"name":"rust-binary","defaultMode":511}},{"name":"seed","emptyDir":{}},{"name":"foo","configMap":{"name":"foo"}},{"name":"bar","configMap":{"name":"bar"}}],"containers":[{"name":"injector","image":"docker.io/library/ubuntu:latest","command":["/zarf-init/zarf-injector","shasum"],"workingDir":"/zarf-init","resources":{"limits":{"cpu":"1","memory":"256Mi"},"requests":{"cpu":"500m","memory":"64Mi"}},"volumeMounts":[{"name":"init","mountPath":"/zarf-init/zarf-injector","subPath":"zarf-injector"},{"name":"seed","mountPath":"/zarf-seed"},{"name":"foo","mountPath":"/zarf-init/foo","subPath":"foo"},{"name":"bar","mountPath":"/zarf-init/bar","subPath":"bar"}],"readinessProbe":{"httpGet":{"path":"/v2/","port":5000},"periodSeconds":2,"successThreshold":1,"failureThreshold":10},"imagePullPolicy":"IfNotPresent"}],"restartPolicy":"Never","nodeName":"injection-node"},"status":{}} diff --git a/src/pkg/layout/package.go b/src/pkg/layout/package.go index decc7a82e1..e68f77b551 100644 --- a/src/pkg/layout/package.go +++ b/src/pkg/layout/package.go @@ -38,13 +38,6 @@ type PackagePaths struct { isLegacyLayout bool } -// InjectionMadnessPaths contains paths for injection madness. -type InjectionMadnessPaths struct { - InjectionBinary string - SeedImagesDir string - InjectorPayloadTarGz string -} - // New returns a new PackagePaths struct. func New(baseDir string) *PackagePaths { return &PackagePaths{ diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 74661b753f..867706d957 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -253,7 +253,7 @@ func (p *Packager) deployInitComponent(ctx context.Context, component types.Zarf // Before deploying the seed registry, start the injector if isSeedRegistry { - err := p.cluster.StartInjectionMadness(ctx, p.layout.Base, p.layout.Images.Base, component.Images) + err := p.cluster.StartInjection(ctx, p.layout.Base, p.layout.Images.Base, component.Images) if err != nil { return nil, err } @@ -266,7 +266,7 @@ func (p *Packager) deployInitComponent(ctx context.Context, component types.Zarf // Do cleanup for when we inject the seed registry during initialization if isSeedRegistry { - if err := p.cluster.StopInjectionMadness(ctx); err != nil { + if err := p.cluster.StopInjection(ctx); err != nil { return nil, fmt.Errorf("unable to seed the Zarf Registry: %w", err) } }