Skip to content

Commit 61e5eec

Browse files
anik120anik120
committed
(e2e) Use custom built test catalog for e2e testing
This PR * Stands up a local image registry in the cluster * Builds bundles and catalog images and uploads them to the image registry * Uses the custom images in the e2e test suite * Also introduces a `pullSecret` field for the Operator API's Spec struct, to allow installation of operators on cluster whose bundles require imagePullSecret to be provisioned. This is required because the bundle images built and uploaded to the local registry above requires a pull secret for the local registry. closes #215
1 parent b982ad0 commit 61e5eec

26 files changed

+14925
-13
lines changed

‎Makefile‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,17 @@ export GORELEASER_VERSION ?= v1.16.2
1111
export RUKPAK_VERSION=$(shell go list -mod=mod -m -f "{{.Version}}" github.com/operator-framework/rukpak)
1212
export WAIT_TIMEOUT ?= 60s
1313
IMG?=$(IMAGE_REPO):$(IMAGE_TAG)
14+
TESTDATA_DIR := testdata
1415

1516
OPERATOR_CONTROLLER_NAMESPACE ?= operator-controller-system
1617
KIND_CLUSTER_NAME ?= operator-controller
1718

19+
REGISTRY_NAME="docker-registry"
20+
REGISTRY_NAMESPACE=operator-controller-e2e
21+
DNS_NAME=$(REGISTRY_NAME).$(REGISTRY_NAMESPACE).svc.cluster.local
22+
23+
CONTAINER_RUNTIME ?= docker
24+
1825
# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
1926
ifeq (,$(shell go env GOBIN))
2027
GOBIN=$(shell go env GOPATH)/bin
@@ -82,7 +89,7 @@ test-unit: envtest ## Run the unit tests
8289
eval $$($(ENVTEST) use -p env $(ENVTEST_VERSION)) && go test -tags $(GO_BUILD_TAGS) -count=1 -short $(UNIT_TEST_DIRS) -coverprofile cover.out
8390

8491
e2e: KIND_CLUSTER_NAME=operator-controller-e2e
85-
e2e: run test-e2e kind-cluster-cleanup ## Run e2e test suite on local kind cluster
92+
e2e: run image-registry kind-load-test-artifacts registry-load-test-artifacts test-e2e kind-cluster-cleanup ## Run e2e test suite on local kind cluster
8693

8794
kind-load: kind ## Loads the currently constructed image onto the cluster
8895
$(KIND) load docker-image $(IMG) --name $(KIND_CLUSTER_NAME)
@@ -94,6 +101,20 @@ kind-cluster: kind kind-cluster-cleanup ## Standup a kind cluster
94101
kind-cluster-cleanup: kind ## Delete the kind cluster
95102
$(KIND) delete cluster --name ${KIND_CLUSTER_NAME}
96103

104+
image-registry: ## Setup in-cluster image registry
105+
./test/tools/imageregistry/setup_imageregistry.sh ${KIND_CLUSTER_NAME}
106+
107+
kind-load-test-artifacts: kind ## Load the e2e testdata container images into a kind cluster
108+
$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.47.0 -t localhost/testdata/bundles/registry-v1/prometheus-operator.v0.47.0
109+
$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/catalogs -f $(TESTDATA_DIR)/catalogs/test-catalog.Dockerfile -t localhost/testdata/catalogs/test-catalog
110+
$(KIND) load docker-image localhost/testdata/bundles/registry-v1/prometheus-operator.v0.47.0 --name $(KIND_CLUSTER_NAME)
111+
$(KIND) load docker-image localhost/testdata/catalogs/test-catalog --name $(KIND_CLUSTER_NAME)
112+
113+
registry-load-test-artifacts: ## Load e2e testdata container images created in kind-load-test-artifacts into registry
114+
$(CONTAINER_RUNTIME) tag localhost/testdata/bundles/registry-v1/prometheus-operator.v0.47.0 $(DNS_NAME):5000/bundles/registry-v1/prometheus-operator.v0.47.0
115+
$(CONTAINER_RUNTIME) tag localhost/testdata/catalogs/test-catalog $(DNS_NAME):5000/catalogs/test-catalog
116+
./test/tools/imageregistry/load_test_image.sh $(KIND) $(KIND_CLUSTER_NAME)
117+
97118
##@ Build
98119

99120
BUILDCMD = sh -c 'mkdir -p $(BUILDBIN) && ${GORELEASER} build ${GORELEASER_ARGS} --single-target -o $(BUILDBIN)/manager'

‎api/v1alpha1/operator_types.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,11 @@ type OperatorSpec struct {
4242
//+kubebuilder:validation:Pattern:=^[a-z0-9]+([\.-][a-z0-9]+)*$
4343
// Channel constraint defintion
4444
Channel string `json:"channel,omitempty"`
45+
46+
//+kubebuilder:validation:MaxLength:=48
47+
//+kubebuilder:validation:Pattern:=^[a-z0-9]+(-[a-z0-9]+)*$
48+
// ImagePullSecretName contains the name of the image pull secret that will be used to provision the operator bundle, in the rukpak-system namespace.
49+
ImagePullSecretName string `json:"pullSecret,omitempty"` // TODO: Find a better place for this field. This is only valid if the package bundles are OCI images.
4550
}
4651

4752
const (

‎config/crd/bases/operators.operatorframework.io_operators.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,13 @@ spec:
4444
maxLength: 48
4545
pattern: ^[a-z0-9]+(-[a-z0-9]+)*$
4646
type: string
47+
pullSecret:
48+
description: ImagePullSecretName contains the name of the image pull
49+
secret that will be used to provision the operator bundle, in the
50+
rukpak-system namespace.
51+
maxLength: 48
52+
pattern: ^[a-z0-9]+(-[a-z0-9]+)*$
53+
type: string
4754
version:
4855
description: "Version is an optional semver constraint on the package
4956
version. If not specified, the latest version available of the package

‎internal/controllers/operator_controller.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,12 @@ func (r *OperatorReconciler) getBundleEntityFromSolution(solution *solver.Soluti
241241
}
242242

243243
func (r *OperatorReconciler) generateExpectedBundleDeployment(o operatorsv1alpha1.Operator, bundlePath string) *unstructured.Unstructured {
244+
bdImage := map[string]interface{}{
245+
"ref": bundlePath,
246+
}
247+
if o.Spec.ImagePullSecretName != "" {
248+
bdImage["pullSecret"] = o.Spec.ImagePullSecretName
249+
}
244250
// We use unstructured here to avoid problems of serializing default values when sending patches to the apiserver.
245251
// If you use a typed object, any default values from that struct get serialized into the JSON patch, which could
246252
// cause unrelated fields to be patched back to the default value even though that isn't the intention. Using an
@@ -261,10 +267,8 @@ func (r *OperatorReconciler) generateExpectedBundleDeployment(o operatorsv1alpha
261267
"provisionerClassName": "core-rukpak-io-registry",
262268
"source": map[string]interface{}{
263269
// TODO: Don't assume image type
264-
"type": string(rukpakv1alpha1.SourceTypeImage),
265-
"image": map[string]interface{}{
266-
"ref": bundlePath,
267-
},
270+
"type": string(rukpakv1alpha1.SourceTypeImage),
271+
"image": bdImage,
268272
},
269273
},
270274
},

‎test/e2e/install_test.go‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,10 @@ import (
1717
)
1818

1919
const (
20-
defaultTimeout = 30 * time.Second
21-
defaultPoll = 1 * time.Second
20+
defaultTimeout = 30 * time.Second
21+
defaultPoll = 1 * time.Second
22+
testCatalogRef = "docker-registry.operator-controller-e2e.svc.cluster.local:5000/catalogs/test-catalog:latest"
23+
localRegistrySecret = "registrysecret"
2224
)
2325

2426
var _ = Describe("Operator Install", func() {
@@ -32,14 +34,15 @@ var _ = Describe("Operator Install", func() {
3234
When("An operator is installed from an operator catalog", func() {
3335
BeforeEach(func() {
3436
ctx = context.Background()
35-
pkgName = "argocd-operator"
37+
pkgName = "prometheus"
3638
operatorName = fmt.Sprintf("operator-%s", rand.String(8))
3739
operator = &operatorv1alpha1.Operator{
3840
ObjectMeta: metav1.ObjectMeta{
3941
Name: operatorName,
4042
},
4143
Spec: operatorv1alpha1.OperatorSpec{
42-
PackageName: pkgName,
44+
PackageName: pkgName,
45+
ImagePullSecretName: localRegistrySecret,
4346
},
4447
}
4548
operatorCatalog = &catalogd.Catalog{
@@ -50,9 +53,8 @@ var _ = Describe("Operator Install", func() {
5053
Source: catalogd.CatalogSource{
5154
Type: catalogd.SourceTypeImage,
5255
Image: &catalogd.ImageSource{
53-
// (TODO): Set up a local image registry, and build and store a test catalog in it
54-
// to use in the test suite
55-
Ref: "quay.io/operatorhubio/catalog:latest",
56+
Ref: testCatalogRef,
57+
PullSecret: localRegistrySecret,
5658
},
5759
},
5860
},
@@ -64,7 +66,7 @@ var _ = Describe("Operator Install", func() {
6466
g.Expect(err).ToNot(HaveOccurred())
6567
g.Expect(len(operatorCatalog.Status.Conditions)).To(Equal(1))
6668
g.Expect(operatorCatalog.Status.Conditions[0].Message).To(ContainSubstring("successfully unpacked the catalog image"))
67-
}).WithTimeout(5 * time.Minute).WithPolling(defaultPoll).Should(Succeed())
69+
}).WithTimeout(1 * time.Minute).WithPolling(defaultPoll).Should(Succeed())
6870
})
6971
It("resolves the specified package with correct bundle path", func() {
7072
By("creating the Operator resource")

‎test/tools/imageregistry/bundle_local_image.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
apiVersion: core.rukpak.io/v1alpha1
2+
kind: Bundle
3+
metadata:
4+
name: prometheus.v0.47.0
5+
spec:
6+
source:
7+
type: image
8+
image:
9+
ref: docker-registry.operator-controller-e2e.svc.cluster.local:5000/bundles/prometheus:0.47.0
10+
pullSecret: registrysecret
11+
provisionerClassName: core-rukpak-io-registry

‎test/tools/imageregistry/daemonset.yaml‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: setup-script
5+
data:
6+
setup.sh: |
7+
echo "$TRUSTED_CERT" > /usr/local/share/ca-certificates/ca.crt && update-ca-certificates && systemctl restart containerd
8+
---
9+
apiVersion: apps/v1
10+
kind: DaemonSet
11+
metadata:
12+
name: node-custom-setup
13+
labels:
14+
k8s-app: node-custom-setup
15+
spec:
16+
selector:
17+
matchLabels:
18+
k8s-app: node-custom-setup
19+
template:
20+
metadata:
21+
labels:
22+
k8s-app: node-custom-setup
23+
spec:
24+
hostPID: true
25+
hostNetwork: true
26+
initContainers:
27+
- name: init-node
28+
command: ["nsenter"]
29+
args: ["--mount=/proc/1/ns/mnt", "--", "sh", "-c", "$(SETUP_SCRIPT)"]
30+
image: debian
31+
env:
32+
- name: TRUSTED_CERT
33+
valueFrom:
34+
configMapKeyRef:
35+
name: trusted-ca
36+
key: ca.crt
37+
- name: SETUP_SCRIPT
38+
valueFrom:
39+
configMapKeyRef:
40+
name: setup-script
41+
key: setup.sh
42+
securityContext:
43+
privileged: true
44+
containers:
45+
- name: wait
46+
image: registry.k8s.io/pause:3.1

‎test/tools/imageregistry/load_test_image.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env bash
2+
3+
export REGISTRY_NAME="docker-registry"
4+
export REGISTRY_NAMESPACE=operator-controller-e2e
5+
export DNS_NAME=$REGISTRY_NAME.$REGISTRY_NAMESPACE.svc.cluster.local
6+
KIND=$1
7+
KIND_CLUSTER_NAME=$2
8+
9+
# push test bundle image into in-cluster docker registry
10+
kubectl exec nerdctl -n $REGISTRY_NAMESPACE -- sh -c "nerdctl login -u myuser -p mypasswd $DNS_NAME:5000 --insecure-registry"
11+
12+
for x in $(docker images --format "{{.Repository}}:{{.Tag}}" | grep $DNS_NAME); do
13+
echo $x
14+
$KIND load docker-image $x --name $KIND_CLUSTER_NAME
15+
kubectl exec nerdctl -n $REGISTRY_NAMESPACE -- sh -c "nerdctl -n k8s.io push $x --insecure-registry"
16+
kubectl exec nerdctl -n $REGISTRY_NAMESPACE -- sh -c "nerdctl -n k8s.io rmi $x --insecure-registry"
17+
done
18+

‎test/tools/imageregistry/nerdctl.yaml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
name: nerdctl
5+
spec:
6+
containers:
7+
- command:
8+
- sleep
9+
- infinity
10+
image: ghcr.io/containerd/nerdctl
11+
imagePullPolicy: Always
12+
name: nerdctl
13+
volumeMounts:
14+
- mountPath: /run/containerd
15+
name: run-containerd
16+
- mountPath: /var/lib/containerd
17+
name: var-lib-containerd
18+
volumes:
19+
- name: run-containerd
20+
hostPath:
21+
path: /run/containerd
22+
- name: var-lib-containerd
23+
hostPath:
24+
path: /var/lib/containerd

‎test/tools/imageregistry/registry.yaml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
name: docker-registry-pod
5+
labels:
6+
app: registry
7+
spec:
8+
initContainers:
9+
- name: auth
10+
image: registry:2.6.2
11+
command:
12+
- "sh"
13+
- "-c"
14+
- "htpasswd -Bbn myuser mypasswd >> /auth/htpasswd"
15+
volumeMounts:
16+
- name: auth-vol
17+
mountPath: "/auth"
18+
containers:
19+
- name: registry
20+
image: registry:2.6.2
21+
volumeMounts:
22+
- name: repo-vol
23+
mountPath: "/var/lib/registry"
24+
- name: certs-vol
25+
mountPath: "/certs"
26+
readOnly: true
27+
- name: auth-vol
28+
mountPath: "/auth"
29+
readOnly: true
30+
env:
31+
- name: REGISTRY_AUTH
32+
value: "htpasswd"
33+
- name: REGISTRY_AUTH_HTPASSWD_REALM
34+
value: "Registry Realm"
35+
- name: REGISTRY_AUTH_HTPASSWD_PATH
36+
value: "/auth/htpasswd"
37+
- name: REGISTRY_HTTP_TLS_CERTIFICATE
38+
value: "/certs/tls.crt"
39+
- name: REGISTRY_HTTP_TLS_KEY
40+
value: "/certs/tls.key"
41+
volumes:
42+
- name: repo-vol
43+
emptyDir: {}
44+
- name: certs-vol
45+
secret:
46+
secretName: certs-secret
47+
- name: auth-vol
48+
emptyDir: {}
49+

0 commit comments

Comments
 (0)