Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add kustomize support for declarative management #952

Merged
merged 1 commit into from
Oct 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .github/workflows/functests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,11 @@ jobs:
while kubectl get pods -n kube-system --no-headers | grep -vqw Running; do echo "Waiting for Minikube pods to be ready"; sleep 3; done
while kubectl get nodes --no-headers | grep -vqw Ready; do echo "Waiting for Minikube node to be ready"; sleep 3; done

- name: Enable kustomize
if: contains(fromJson('["v1.24.17", "v1.25.16", "v1.26.15", "v1.27.15", "v1.28.11", "v1.29.6", "v1.30.2"]'), matrix.kube-version)
run: |
echo "KUSTOMIZE=true" >> $GITHUB_ENV

- name: Run tests
run: |
functests/run-tests.sh
Expand Down
68 changes: 68 additions & 0 deletions functests/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,41 @@ function install_directpv() {
sleep 10
}

# install_directpv_kustomize <plugin> <pod_count>
function install_directpv_kustomize() {
directpv_client="$1"
echo "* Installing DirectPV via kustomize"

cat > kustomization.yaml <<EOF
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

resources:
- ../resources/base

images:
- name: quay.io/minio/directpv
newTag: ${VERSION}
EOF

kubectl kustomize | kubectl apply -f -

required_count="$2"
running_count=0
while [[ $running_count -lt $required_count ]]; do
echo " ...waiting for $(( required_count - running_count )) DirectPV pods to come up"
sleep 1m
running_count=$(kubectl get pods --field-selector=status.phase=Running --no-headers --namespace=directpv | wc -l)
done

while ! "${directpv_client}" info --quiet; do
echo " ...waiting for DirectPV to come up"
sleep 1m
done

sleep 10
}

# uninstall_directpv <plugin> <pod_count>
function uninstall_directpv() {
directpv_client="$1"
Expand All @@ -160,6 +195,39 @@ function uninstall_directpv() {
return 0
}

# uninstall_directpv_kustomize <pod_count>
function uninstall_directpv_kustomize() {
echo "* Uninstalling DirectPV via kustomize"

cat > kustomization.yaml <<EOF
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

resources:
- ../resources/base

images:
- name: quay.io/minio/directpv
newTag: ${VERSION}
EOF
# delete hangs sometimes even after the resources are deleted
kubectl kustomize | timeout 15s kubectl delete -f -

pending="$2"
while [[ $pending -gt 0 ]]; do
echo " ...waiting for ${pending} DirectPV pods to go down"
sleep 5
pending=$(kubectl get pods --field-selector=status.phase=Running --no-headers --namespace=directpv-min-io 2>/dev/null | wc -l)
done

while kubectl get namespace directpv-min-io --no-headers 2>/dev/null | grep -q .; do
echo " ...waiting for directpv-min-io namespace to be removed"
sleep 5
done

return 0
}

# usage: check_drives_status <plugin>
function check_drives_status() {
if ! is_github_workflow; then
Expand Down
12 changes: 10 additions & 2 deletions functests/tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,11 @@ function run_tests() {
setup_lvm
setup_luks
pod_count=$(( 3 + ACTIVE_NODES ))
install_directpv "${DIRECTPV_DIR}/kubectl-directpv" "${pod_count}"
if [ "${KUSTOMIZE}" == "true" ]; then
install_directpv_kustomize "${DIRECTPV_DIR}/kubectl-directpv" "${pod_count}"
else
install_directpv "${DIRECTPV_DIR}/kubectl-directpv" "${pod_count}"
fi
add_drives "${DIRECTPV_DIR}/kubectl-directpv"
deploy_minio minio.yaml
test_force_delete
Expand All @@ -34,7 +38,11 @@ function run_tests() {
uninstall_minio "${DIRECTPV_DIR}/kubectl-directpv" minio.yaml
test_volume_expansion "${DIRECTPV_DIR}/kubectl-directpv" sleep.yaml
remove_drives "${DIRECTPV_DIR}/kubectl-directpv"
uninstall_directpv "${DIRECTPV_DIR}/kubectl-directpv" "${pod_count}"
if [ "${KUSTOMIZE}" == "true" ]; then
uninstall_directpv_kustomize "${pod_count}"
else
uninstall_directpv "${DIRECTPV_DIR}/kubectl-directpv" "${pod_count}"
fi
unmount_directpv
remove_luks
remove_lvm
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
annotations:
directpv.min.io/plugin-version: v4.0.10
creationTimestamp: null
labels:
application-name: directpv.min.io
Expand Down
21 changes: 0 additions & 21 deletions operator/helm-charts/directpv-chart/templates/ClusterRole.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
annotations:
directpv.min.io/plugin-version: v4.0.10
rbac.authorization.kubernetes.io/autoupdate: "true"
creationTimestamp: null
labels:
Expand Down Expand Up @@ -63,20 +62,6 @@ rules:
- patch
- update
- watch
- apiGroups:
- snapshot.storage.k8s.io
resources:
- volumesnapshots
verbs:
- get
- list
- apiGroups:
- snapshot.storage.k8s.io
resources:
- volumesnapshotcontents
verbs:
- get
- list
- apiGroups:
- storage.k8s.io
resources:
Expand Down Expand Up @@ -125,10 +110,8 @@ rules:
- watch
- apiGroups:
- apiextensions.k8s.io
- directpv.min.io
resources:
- customresourcedefinitions
- customresourcedefinition
verbs:
- create
- delete
Expand All @@ -155,7 +138,6 @@ rules:
- ""
resources:
- pods
- pod
verbs:
- get
- list
Expand All @@ -164,10 +146,7 @@ rules:
- ""
resources:
- secrets
- secret
verbs:
- get
- list
- watch


Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
annotations:
directpv.min.io/plugin-version: v4.0.10
rbac.authorization.kubernetes.io/autoupdate: "true"
creationTimestamp: null
labels:
Expand Down
3 changes: 0 additions & 3 deletions operator/helm-charts/directpv-chart/templates/DaemonSet.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,6 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
annotations:
directpv.min.io/image-tag: v4.0.10
directpv.min.io/plugin-version: v4.0.10
creationTimestamp: null
labels:
application-name: directpv.min.io
Expand Down
3 changes: 0 additions & 3 deletions operator/helm-charts/directpv-chart/templates/Deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,6 @@
apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
directpv.min.io/image-tag: v4.0.10
directpv.min.io/plugin-version: v4.0.10
creationTimestamp: null
labels:
application-name: directpv.min.io
Expand Down
2 changes: 0 additions & 2 deletions operator/helm-charts/directpv-chart/templates/Namespace.yaml
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
apiVersion: v1
kind: Namespace
metadata:
annotations:
directpv.min.io/plugin-version: v4.0.10
creationTimestamp: null
finalizers:
- foregroundDeletion
Expand Down
1 change: 0 additions & 1 deletion operator/helm-charts/directpv-chart/templates/Role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
annotations:
directpv.min.io/plugin-version: v4.0.10
rbac.authorization.kubernetes.io/autoupdate: "true"
creationTimestamp: null
labels:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
annotations:
directpv.min.io/plugin-version: v4.0.10
rbac.authorization.kubernetes.io/autoupdate: "true"
creationTimestamp: null
labels:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ apiVersion: v1
kind: ServiceAccount
metadata:
annotations:
directpv.min.io/plugin-version: v4.0.10
creationTimestamp: null
labels:
application-name: directpv.min.io
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ allowedTopologies:
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
annotations:
directpv.min.io/plugin-version: v4.0.10
creationTimestamp: null
finalizers:
- foregroundDeletion
Expand Down
10 changes: 0 additions & 10 deletions pkg/admin/installer/args.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"fmt"
"io"
"path"
"regexp"

"github.com/minio/directpv/pkg/utils"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -80,18 +79,10 @@ type Args struct {
nodeDriverRegistrarImage string
livenessProbeImage string
csiResizerImage string
imageTag string
}

var imageTagRegex = regexp.MustCompile(`:([^/]+)$`)

// NewArgs creates arguments for DirectPV installation.
func NewArgs(image string) *Args {
imageTag := "dev"
matchIndex := imageTagRegex.FindStringSubmatchIndex(image)
if len(matchIndex) > 0 && len(image) > matchIndex[0]+1 {
imageTag = image[matchIndex[0]+1:]
}
return &Args{
image: image,
Registry: "quay.io",
Expand All @@ -101,7 +92,6 @@ func NewArgs(image string) *Args {
nodeDriverRegistrarImage: nodeDriverRegistrarImage,
livenessProbeImage: livenessProbeImage,
csiResizerImage: csiResizerImage,
imageTag: imageTag,
}
}

Expand Down
11 changes: 2 additions & 9 deletions pkg/admin/installer/csidriver.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"errors"
"fmt"

directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types"
"github.com/minio/directpv/pkg/client"
"github.com/minio/directpv/pkg/consts"
legacyclient "github.com/minio/directpv/pkg/legacy/client"
Expand Down Expand Up @@ -95,10 +94,7 @@ func (t csiDriverTask) doCreateCSIDriver(ctx context.Context, args *Args, versio
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: metav1.NamespaceNone,
Annotations: map[string]string{
string(directpvtypes.PluginVersionLabelKey): args.PluginVersion,
},
Labels: defaultLabels,
Labels: defaultLabels,
},
Spec: storagev1.CSIDriverSpec{
PodInfoOnMount: &podInfoOnMount,
Expand Down Expand Up @@ -128,10 +124,7 @@ func (t csiDriverTask) doCreateCSIDriver(ctx context.Context, args *Args, versio
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: metav1.NamespaceNone,
Annotations: map[string]string{
string(directpvtypes.PluginVersionLabelKey): args.PluginVersion,
},
Labels: defaultLabels,
Labels: defaultLabels,
},
Spec: storagev1beta1.CSIDriverSpec{
PodInfoOnMount: &podInfoOnMount,
Expand Down
11 changes: 3 additions & 8 deletions pkg/admin/installer/daemonset.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"context"
"fmt"

directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types"
"github.com/minio/directpv/pkg/client"
"github.com/minio/directpv/pkg/consts"
"github.com/minio/directpv/pkg/k8s"
Expand Down Expand Up @@ -215,11 +214,7 @@ func newDaemonset(podSpec corev1.PodSpec, name, selectorValue string, args *Args
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Annotations: map[string]string{
string(directpvtypes.ImageTagLabelKey): args.imageTag,
string(directpvtypes.PluginVersionLabelKey): args.PluginVersion,
},
Labels: defaultLabels,
Labels: defaultLabels,
},
Spec: appsv1.DaemonSetSpec{
Selector: metav1.AddLabelToSelector(&metav1.LabelSelector{}, selectorKey, selectorValue),
Expand Down Expand Up @@ -297,7 +292,7 @@ func (t daemonsetTask) doCreateDaemonset(ctx context.Context, args *Args) (err e
}
}
if selectorValue == "" {
selectorValue = fmt.Sprintf("%v-%v", consts.Identity, getRandSuffix())
selectorValue = fmt.Sprintf("%v-%v", consts.Identity, consts.NodeServerName)
}

daemonset := newDaemonset(podSpec, consts.NodeServerName, selectorValue, args)
Expand Down Expand Up @@ -360,7 +355,7 @@ func (t daemonsetTask) doCreateLegacyDaemonset(ctx context.Context, args *Args)
}
}
if selectorValue == "" {
selectorValue = fmt.Sprintf("%v-%v", consts.Identity, getRandSuffix())
selectorValue = fmt.Sprintf("%v-%v", consts.Identity, consts.LegacyNodeServerName)
}

daemonset := newDaemonset(podSpec, consts.LegacyNodeServerName, selectorValue, args)
Expand Down
9 changes: 2 additions & 7 deletions pkg/admin/installer/deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"context"
"fmt"

directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types"
"github.com/minio/directpv/pkg/client"
"github.com/minio/directpv/pkg/consts"
"github.com/minio/directpv/pkg/k8s"
Expand Down Expand Up @@ -195,7 +194,7 @@ func (t deploymentTask) doCreateDeployment(ctx context.Context, args *Args, lega
}
}
if selectorValue == "" {
selectorValue = fmt.Sprintf("%v-%v", consts.ControllerServerName, getRandSuffix())
selectorValue = fmt.Sprintf("%v-%v", consts.ControllerServerName, name)
}

replicas := int32(3)
Expand All @@ -207,11 +206,7 @@ func (t deploymentTask) doCreateDeployment(ctx context.Context, args *Args, lega
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Annotations: map[string]string{
string(directpvtypes.ImageTagLabelKey): args.imageTag,
string(directpvtypes.PluginVersionLabelKey): args.PluginVersion,
},
Labels: defaultLabels,
Labels: defaultLabels,
},
Spec: appsv1.DeploymentSpec{
Replicas: &replicas,
Expand Down
Loading