Add basic e2e test for extensions

Signed-off-by: Natalie Arellano <narellano@vmware.com>

DEVELOPMENT.md

@@ -91,12 +91,6 @@ IMAGE_REGISTRY=gcr.io/<some-project> \
   make e2e
 ```
 
-* The IMAGE_REGISTRY environment variable must point at a registry with local write access - e.g.
-
-```bash
-export IMAGE_REGISTRY="gcr.io/<some-project>"
-```
-
 * The KPACK_TEST_NAMESPACE_LABELS environment variable allows you to define additional labels for the test namespace, e.g.
 
 ```bash

Makefile

@@ -10,6 +10,6 @@ unit-ci:
 	$(GOCMD) test ./pkg/... -coverprofile=coverage.txt -covermode=atomic
 
 e2e:
-	$(GOCMD) test --timeout=30m -v ./test/...
+	$(GOCMD) test --timeout=30m -failfast -v ./test/...
 
 .PHONY: unit unit-ci e2e

cmd/controller/main.go

@@ -50,8 +50,10 @@ import (
 	"github.com/pivotal/kpack/pkg/reconciler/buildpack"
 	"github.com/pivotal/kpack/pkg/reconciler/clusterbuilder"
 	"github.com/pivotal/kpack/pkg/reconciler/clusterbuildpack"
+	"github.com/pivotal/kpack/pkg/reconciler/clusterextension"
 	"github.com/pivotal/kpack/pkg/reconciler/clusterstack"
 	"github.com/pivotal/kpack/pkg/reconciler/clusterstore"
+	"github.com/pivotal/kpack/pkg/reconciler/extension"
 	"github.com/pivotal/kpack/pkg/reconciler/image"
 	"github.com/pivotal/kpack/pkg/reconciler/lifecycle"
 	"github.com/pivotal/kpack/pkg/reconciler/sourceresolver"
@@ -207,8 +209,10 @@ func main() {
 	sourceResolverController := sourceresolver.NewController(ctx, options, sourceResolverInformer, gitResolver, blobResolver, registryResolver)
 	builderController, builderResync := builder.NewController(ctx, options, builderInformer, builderCreator, keychainFactory, clusterStoreInformer, buildpackInformer, clusterBuildpackInformer, clusterStackInformer, extensionInformer, clusterExtensionInformer)
 	buildpackController := buildpack.NewController(ctx, options, keychainFactory, buildpackInformer, remoteStoreReader)
+	extensionController := extension.NewController(ctx, options, keychainFactory, extensionInformer, remoteStoreReader)
 	clusterBuilderController, clusterBuilderResync := clusterbuilder.NewController(ctx, options, clusterBuilderInformer, builderCreator, keychainFactory, clusterStoreInformer, clusterBuildpackInformer, clusterStackInformer, clusterExtensionInformer)
 	clusterBuildpackController := clusterbuildpack.NewController(ctx, options, keychainFactory, clusterBuildpackInformer, remoteStoreReader)
+	clusterExtensionController := clusterextension.NewController(ctx, options, keychainFactory, clusterExtensionInformer, remoteStoreReader)
 	clusterStoreController := clusterstore.NewController(ctx, options, keychainFactory, clusterStoreInformer, remoteStoreReader)
 	clusterStackController := clusterstack.NewController(ctx, options, keychainFactory, clusterStackInformer, remoteStackReader)
 	lifecycleController := lifecycle.NewController(ctx, options, k8sClient, config.LifecycleConfigName, lifecycleConfigmapInformer, lifecycleProvider)
@@ -243,8 +247,10 @@ func main() {
 		run(buildController, routinesPerController),
 		run(builderController, routinesPerController),
 		run(buildpackController, routinesPerController),
+		run(extensionController, routinesPerController),
 		run(clusterBuilderController, routinesPerController),
 		run(clusterBuildpackController, routinesPerController),
+		run(clusterExtensionController, routinesPerController),
 		run(clusterStoreController, routinesPerController),
 		run(lifecycleController, routinesPerController),
 		run(sourceResolverController, 2*routinesPerController),

config/clusterextension.yaml

@@ -0,0 +1,31 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: clusterextensions.kpack.io
+spec:
+  group: kpack.io
+  versions:
+  - name: v1alpha2
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        x-kubernetes-preserve-unknown-fields: true
+    subresources:
+      status: {}
+    additionalPrinterColumns:
+    - name: Ready
+      type: string
+      jsonPath: ".status.conditions[?(@.type==\"Ready\")].status"
+  names:
+    kind: ClusterExtension
+    listKind: ClusterExtensionList
+    singular: clusterextension
+    plural: clusterextensions
+    shortNames:
+    - clstext
+    - clstexts
+    categories:
+    - kpack
+  scope: Cluster

config/controllerrole.yaml

@@ -23,10 +23,14 @@ rules:
   - builders/status
   - buildpacks
   - buildpacks/status
+  - extensions
+  - extensions/status
   - clusterbuilders
   - clusterbuilders/status
   - clusterbuildpacks
   - clusterbuildpacks/status
+  - clusterextensions
+  - clusterextensions/status
   - clusterstores
   - clusterstores/status
   - clusterstacks

config/extension.yaml

@@ -0,0 +1,32 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: extensions.kpack.io
+spec:
+  group: kpack.io
+  versions:
+  - name: v1alpha2
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        x-kubernetes-preserve-unknown-fields: true
+    subresources:
+      status: {}
+    additionalPrinterColumns:
+    - name: Ready
+      type: string
+      jsonPath: ".status.conditions[?(@.type==\"Ready\")].status"
+  names:
+    kind: Extension
+    listKind: ExtensionList
+    singular: extension
+    plural: extensions
+    shortNames:
+    - ext
+    - exts
+    categories:
+    - kpack
+  scope: Namespaced
+

go.sum

@@ -58,7 +58,7 @@ dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek=
 filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
-github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230618160516-e936619f9f18 h1:rd389Q26LMy03gG4anandGFC2LW/xvjga5GezeeaxQk=
+github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230919221257-8b5d3ce2d11d h1:zjqpY4C7H15HjRPEenkS4SAn3Jy2eRRjkjZbGR30TOg=
 github.com/AliyunContainerService/ack-ram-tool/pkg/credentials/alibabacloudsdkgo/helper v0.2.0 h1:8+4G8JaejP8Xa6W46PzJEwisNgBXMvFcz78N6zG/ARw=
 github.com/AliyunContainerService/ack-ram-tool/pkg/credentials/alibabacloudsdkgo/helper v0.2.0/go.mod h1:GgeIE+1be8Ivm7Sh4RgwI42aTtC9qrcj+Y9Y6CjJhJs=
 github.com/Azure/azure-sdk-for-go v55.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=

pkg/apis/build/v1alpha2/build_pod.go

@@ -126,12 +126,13 @@ func (c BuildContext) os() string {
 }
 
 type BuildPodBuilderConfig struct {
-	StackID      string
-	RunImage     string
-	Uid          int64
-	Gid          int64
-	PlatformAPIs []string
-	OS           string
+	StackID       string
+	RunImage      string
+	Uid           int64
+	Gid           int64
+	PlatformAPIs  []string
+	OS            string
+	HasExtensions bool
 }
 
 var (
@@ -629,6 +630,10 @@ func (b *Build) BuildPod(images BuildPodImages, buildContext BuildContext) (*cor
 		},
 	}
 
+	if buildContext.BuildPodBuilderConfig.HasExtensions && buildContext.os() != "windows" {
+		b.useImageExtensions(pod)
+	}
+
 	if buildContext.InjectedSidecarSupport && buildContext.os() != "windows" {
 		pod = b.useStandardContainers(images.BuildWaiterImage, pod)
 	}
@@ -712,8 +717,42 @@ func setUpBuildWaiter(container corev1.Container, waitFile string) corev1.Contai
 
 }
 
-func (b *Build) useStandardContainers(buildWaiterImage string, pod *corev1.Pod) *corev1.Pod {
+func (b *Build) useImageExtensions(pod *corev1.Pod) {
+	lifecycleExperimentalEnvVar := corev1.EnvVar{Name: "CNB_EXPERIMENTAL_MODE", Value: "warn"}
 
+	kanikoVolume := corev1.Volume{
+		Name: "kaniko",
+		VolumeSource: corev1.VolumeSource{
+			EmptyDir: &corev1.EmptyDirVolumeSource{},
+		},
+	}
+	pod.Spec.Volumes = append(pod.Spec.Volumes, kanikoVolume)
+	kanikoMount := corev1.VolumeMount{
+		Name:      "kaniko",
+		MountPath: "/kaniko",
+	}
+
+	for idx, container := range pod.Spec.InitContainers {
+		container.Env = append(container.Env, lifecycleExperimentalEnvVar)
+		switch container.Name {
+		case RestoreContainerName:
+			container.VolumeMounts = append(container.VolumeMounts, kanikoMount)
+			container.Args = append(container.Args, fmt.Sprintf("-build-image=%s", b.Spec.Builder.Image))
+		case BuildContainerName:
+			runAsNonRoot := false
+			rootUser := int64(0)
+			container.Name = ExtendContainerName
+			container.Command = []string{"/cnb/lifecycle/extender"}
+			container.VolumeMounts = append(container.VolumeMounts, kanikoMount)
+			container.SecurityContext.RunAsNonRoot = &runAsNonRoot
+			container.SecurityContext.RunAsUser = &rootUser
+		}
+		pod.Spec.InitContainers[idx] = container
+	}
+
+}
+
+func (b *Build) useStandardContainers(buildWaiterImage string, pod *corev1.Pod) *corev1.Pod {
 	containers := pod.Spec.InitContainers
 	pod.Spec.InitContainers = []corev1.Container{
 		{
@@ -1085,7 +1124,11 @@ func (b *Build) setupCosignVolumes(secrets []corev1.Secret) ([]corev1.Volume, []
 }
 
 var (
-	supportedPlatformAPIVersions = []*semver.Version{semver.MustParse("0.9"), semver.MustParse("0.8"), semver.MustParse("0.7")}
+	supportedPlatformAPIVersions = []*semver.Version{
+		semver.MustParse("0.9"),
+		semver.MustParse("0.8"),
+		semver.MustParse("0.7"),
+	}
 )
 
 func (bc BuildContext) highestSupportedPlatformAPI(b *Build) (*semver.Version, error) {

pkg/apis/build/v1alpha2/build_pod_test.go

@@ -889,6 +889,7 @@ func testBuildPod(t *testing.T, when spec.G, it spec.S) {
 				"someimage/name:tag3",
 			}, pod.Spec.InitContainers[5].Args)
 		})
+
 		it("configures export step with non-web default process", func() {
 			build.Spec.DefaultProcess = "sys-info"
 			pod, err := build.BuildPod(config, buildContext)
@@ -2460,6 +2461,89 @@ func testBuildPod(t *testing.T, when spec.G, it spec.S) {
 			})
 		})
 
+		when("builder has extensions", func() {
+			it.Before(func() {
+				buildContext.BuildPodBuilderConfig.HasExtensions = true
+			})
+
+			it("sets CNB_EXPERIMENTAL_MODE=warn in the lifecycle env", func() {
+				pod, err := build.BuildPod(config, buildContext)
+				require.NoError(t, err)
+
+				for _, container := range pod.Spec.InitContainers {
+					assert.Contains(t, container.Env,
+						corev1.EnvVar{
+							Name:  "CNB_EXPERIMENTAL_MODE",
+							Value: "warn",
+						},
+					)
+				}
+			})
+
+			it("provides -build-image to the restorer", func() {
+				pod, err := build.BuildPod(config, buildContext)
+				require.NoError(t, err)
+
+				assert.Contains(t, pod.Spec.InitContainers[3].Args, "-build-image="+builderImage)
+			})
+
+			it("adds kaniko volume to pod and mounts it during restore and extend", func() {
+				pod, err := build.BuildPod(config, buildContext)
+				require.NoError(t, err)
+
+				assert.Contains(t, pod.Spec.Volumes, corev1.Volume{
+					Name:         "kaniko",
+					VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}},
+				})
+
+				for _, container := range pod.Spec.InitContainers {
+					switch container.Name {
+					case buildapi.RestoreContainerName, buildapi.ExtendContainerName:
+						assert.Contains(t, container.VolumeMounts, corev1.VolumeMount{
+							Name:      "kaniko",
+							MountPath: "/kaniko",
+						})
+					default:
+						assert.NotContains(t, container.VolumeMounts, corev1.VolumeMount{
+							Name:      "kaniko",
+							MountPath: "/kaniko",
+						})
+					}
+				}
+			})
+
+			it("runs the extender (as root) instead of the builder", func() {
+				pod, err := build.BuildPod(config, buildContext)
+				require.NoError(t, err)
+
+				assert.Equal(t, buildapi.ExtendContainerName, pod.Spec.InitContainers[4].Name)
+				assert.Equal(t, []string{"/cnb/lifecycle/extender"}, pod.Spec.InitContainers[4].Command)
+
+				for _, container := range pod.Spec.InitContainers {
+					actualRunAsNonRoot := container.SecurityContext.RunAsNonRoot
+					actualRunAsUser := container.SecurityContext.RunAsUser
+					switch container.Name {
+					case buildapi.ExtendContainerName:
+						assert.Equal(t, false, *actualRunAsNonRoot)
+						assert.Equal(t, int64(0), *actualRunAsUser)
+					default:
+						assert.Equal(t, true, *actualRunAsNonRoot)
+						assert.NotEqual(t, nil, actualRunAsUser) // in real life this would be the user from the builder
+					}
+				}
+			})
+
+			it("is possible to use standard containers", func() {
+				buildContext.InjectedSidecarSupport = true
+				config.BuildWaiterImage = "some-image"
+
+				pod, err := build.BuildPod(config, buildContext)
+				require.NoError(t, err)
+
+				assert.Equal(t, buildapi.ExtendContainerName, pod.Spec.Containers[4].Name)
+			})
+		})
+
 		when("complying with the restricted pod security standard", func() {
 			// enforces https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted
 			var pod *corev1.Pod

pkg/apis/build/v1alpha2/builder_conversion.go

@@ -82,7 +82,7 @@ func (bs *NamespacedBuilderSpec) convertFrom(from *v1alpha1.NamespacedBuilderSpe
 
 func (bst *BuilderStatus) convertFrom(from *v1alpha1.BuilderStatus) {
 	bst.Status = from.Status
-	bst.BuilderMetadata = from.BuilderMetadata
+	bst.BuilderMetadataBuildpacks = from.BuilderMetadata
 	bst.Order = from.Order
 	bst.Stack = from.Stack
 	bst.LatestImage = from.LatestImage
@@ -93,7 +93,7 @@ func (bst *BuilderStatus) convertFrom(from *v1alpha1.BuilderStatus) {
 
 func (bst *BuilderStatus) convertTo(to *v1alpha1.BuilderStatus) {
 	to.Status = bst.Status
-	to.BuilderMetadata = bst.BuilderMetadata
+	to.BuilderMetadata = bst.BuilderMetadataBuildpacks
 	to.Order = bst.Order
 	to.Stack = bst.Stack
 	to.LatestImage = bst.LatestImage

pkg/apis/build/v1alpha2/builder_conversion_test.go

@@ -58,9 +58,9 @@ func testBuilderConversion(t *testing.T, when spec.G, it spec.S) {
 				ServiceAccountName: "some-service-account",
 			},
 			Status: BuilderStatus{
-				Status:          corev1alpha1.Status{Conditions: corev1alpha1.Conditions{{Type: "some-type"}}},
-				BuilderMetadata: nil,
-				Order:           nil,
+				Status:                    corev1alpha1.Status{Conditions: corev1alpha1.Conditions{{Type: "some-type"}}},
+				BuilderMetadataBuildpacks: nil,
+				Order:                     nil,
 				Stack: corev1alpha1.BuildStack{
 					RunImage: "",
 					ID:       "",

pkg/apis/build/v1alpha2/builder_lifecycle.go

@@ -12,6 +12,7 @@ type BuilderRecord struct {
 	Image                   string
 	Stack                   corev1alpha1.BuildStack
 	Buildpacks              corev1alpha1.BuildpackMetadataList
+	Extensions              corev1alpha1.BuildpackMetadataList
 	Order                   []corev1alpha1.OrderEntry
 	OrderExtensions         []corev1alpha1.OrderEntry
 	ObservedStoreGeneration int64
@@ -21,7 +22,8 @@ type BuilderRecord struct {
 
 func (bs *BuilderStatus) BuilderRecord(record BuilderRecord) {
 	bs.Stack = record.Stack
-	bs.BuilderMetadata = record.Buildpacks
+	bs.BuilderMetadataBuildpacks = record.Buildpacks
+	bs.BuilderMetadataExtensions = record.Extensions
 	bs.LatestImage = record.Image
 	bs.Conditions = corev1alpha1.Conditions{
 		{