Skip to content

Commit

Permalink
New validation hook to check if scale target is already managed
Browse files Browse the repository at this point in the history
Signed-off-by: Jorge Turrado <jorge_turrado@hotmail.es>
  • Loading branch information
JorTurFer committed Dec 21, 2022
1 parent f4bee67 commit 7756767
Show file tree
Hide file tree
Showing 124 changed files with 16,991 additions and 250 deletions.
13 changes: 8 additions & 5 deletions .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,12 @@ RUN apt-get update \
# Enable go modules
ENV GO111MODULE=on

ENV OPERATOR_RELEASE_VERSION=v1.0.1
RUN curl -LO https://github.com/operator-framework/operator-sdk/releases/download/${OPERATOR_RELEASE_VERSION}/operator-sdk-${OPERATOR_RELEASE_VERSION}-x86_64-linux-gnu \
&& chmod +x operator-sdk-${OPERATOR_RELEASE_VERSION}-x86_64-linux-gnu \
ENV OPERATOR_RELEASE_VERSION=v1.26.0
RUN ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac) \
&& OS=$(uname | awk '{print tolower($0)}') \
&& OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/download/${OPERATOR_RELEASE_VERSION} \
&& curl -LO ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH} \
&& chmod +x operator-sdk_${OS}_${ARCH} \
&& mkdir -p /usr/local/bin/ \
&& cp operator-sdk-${OPERATOR_RELEASE_VERSION}-x86_64-linux-gnu /usr/local/bin/operator-sdk \
&& rm operator-sdk-${OPERATOR_RELEASE_VERSION}-x86_64-linux-gnu
&& cp operator-sdk_${OS}_${ARCH} /usr/local/bin/operator-sdk \
&& rm operator-sdk_${OS}_${ARCH}
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,9 @@ apiserver.local.config/

cover.out

# GO debug binary
# GO debug binaries
cmd/manager/debug.test
__debug_bin

# GO Test result
report.xml
4 changes: 2 additions & 2 deletions BUILD.md
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ Follow these instructions if you want to debug the KEDA operator using VS Code.
"type": "go",
"request": "launch",
"mode": "debug",
"program": "${workspaceFolder}/main.go",
"program": "${workspaceFolder}/cmd/operator/main.go",
"env": {
"WATCH_NAMESPACE": "",
"KEDA_CLUSTER_OBJECT_NAMESPACE": "keda"
Expand Down Expand Up @@ -173,7 +173,7 @@ Follow these instructions if you want to debug the KEDA metrics server using VS
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/adapter/main.go",
"program": "${workspaceFolder}/cmd/adapter/main.go",
"env": {
"WATCH_NAMESPACE": "",
"KEDA_CLUSTER_OBJECT_NAMESPACE": "keda"
Expand Down
3 changes: 1 addition & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@ COPY Makefile Makefile
# Copy the go source
COPY hack/ hack/
COPY version/ version/
COPY main.go main.go
COPY adapter/ adapter/
COPY cmd/ cmd/
COPY apis/ apis/
COPY controllers/ controllers/
COPY pkg/ pkg/
Expand Down
3 changes: 1 addition & 2 deletions Dockerfile.adapter
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@ COPY Makefile Makefile
# Copy the go source
COPY hack/ hack/
COPY version/ version/
COPY main.go main.go
COPY adapter/ adapter/
COPY cmd/ cmd/
COPY apis/ apis/
COPY controllers/ controllers/
COPY pkg/ pkg/
Expand Down
37 changes: 37 additions & 0 deletions Dockerfile.webhooks
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Build the manager binary
FROM --platform=$BUILDPLATFORM ghcr.io/kedacore/build-tools:1.18.8 AS builder

ARG BUILD_VERSION=main
ARG GIT_COMMIT=HEAD
ARG GIT_VERSION=main

WORKDIR /workspace

COPY Makefile Makefile

# Copy the go source
COPY hack/ hack/
COPY version/ version/
COPY cmd/ cmd/
COPY apis/ apis/
COPY controllers/ controllers/
COPY pkg/ pkg/
COPY vendor/ vendor/
COPY go.mod go.mod
COPY go.sum go.sum

# Build
# https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/
ARG TARGETOS
ARG TARGETARCH
RUN VERSION=${BUILD_VERSION} GIT_COMMIT=${GIT_COMMIT} GIT_VERSION=${GIT_VERSION} TARGET_OS=$TARGETOS ARCH=$TARGETARCH make webhooks

# Use distroless as minimal base image to package the manager binary
# Refer to https://github.com/GoogleContainerTools/distroless for more details
FROM gcr.io/distroless/static:nonroot
WORKDIR /
COPY --from=builder /workspace/bin/keda-webhooks .
# 65532 is numeric for nonroot
USER 65532:65532

ENTRYPOINT ["/keda-webhooks", "--zap-log-level=info", "--zap-encoder=console"]
25 changes: 21 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ IMAGE_REPO ?= kedacore

IMAGE_CONTROLLER = $(IMAGE_REGISTRY)/$(IMAGE_REPO)/keda$(SUFFIX):$(VERSION)
IMAGE_ADAPTER = $(IMAGE_REGISTRY)/$(IMAGE_REPO)/keda-metrics-apiserver$(SUFFIX):$(VERSION)
IMAGE_WEBHOOKS = $(IMAGE_REGISTRY)/$(IMAGE_REPO)/keda-webhooks$(SUFFIX):$(VERSION)

BUILD_TOOLS_GO_VERSION = 1.18.8
IMAGE_BUILD_TOOLS = $(IMAGE_REGISTRY)/$(IMAGE_REPO)/build-tools:$(BUILD_TOOLS_GO_VERSION)
Expand Down Expand Up @@ -156,8 +157,10 @@ pkg/mock/mock_scaling/mock_executor/mock_interface.go: pkg/scaling/executor/scal
pkg/mock/mock_scaler/mock_scaler.go: pkg/scalers/scaler.go
$(MOCKGEN) -destination=$@ -package=mock_scalers -source=$^
pkg/mock/mock_scale/mock_interfaces.go: vendor/k8s.io/client-go/scale/interfaces.go
mkdir -p pkg/mock/mock_scale
$(MOCKGEN) k8s.io/client-go/scale ScalesGetter,ScaleInterface > $@
pkg/mock/mock_client/mock_interfaces.go: vendor/sigs.k8s.io/controller-runtime/pkg/client/interfaces.go
mkdir -p pkg/mock/mock_client
$(MOCKGEN) sigs.k8s.io/controller-runtime/pkg/client Patch,Reader,Writer,StatusClient,StatusWriter,Client,WithWatch,FieldIndexer > $@
pkg/scalers/liiklus/mocks/mock_liiklus.go:
$(MOCKGEN) -destination=$@ github.com/kedacore/keda/v2/pkg/scalers/liiklus LiiklusServiceClient
Expand All @@ -168,38 +171,48 @@ pkg/scalers/liiklus/mocks/mock_liiklus.go:

##@ Build

build: generate fmt vet manager adapter ## Build Operator (manager) and Metrics Server (adapter) binaries.
build: generate fmt vet manager adapter webhooks ## Build Operator (manager), Metrics Server (adapter) and Admision Web Hooks (webhooks) binaries.

manager: generate
${GO_BUILD_VARS} go build -ldflags $(GO_LDFLAGS) -mod=vendor -o bin/keda main.go
${GO_BUILD_VARS} go build -ldflags $(GO_LDFLAGS) -mod=vendor -o bin/keda cmd/operator/main.go

adapter: generate
${GO_BUILD_VARS} go build -ldflags $(GO_LDFLAGS) -mod=vendor -o bin/keda-adapter adapter/main.go
${GO_BUILD_VARS} go build -ldflags $(GO_LDFLAGS) -mod=vendor -o bin/keda-adapter cmd/adapter/main.go

webhooks: generate
${GO_BUILD_VARS} go build -ldflags $(GO_LDFLAGS) -mod=vendor -o bin/keda-webhooks cmd/webhooks/main.go

run: manifests generate ## Run a controller from your host.
WATCH_NAMESPACE="" go run -ldflags $(GO_LDFLAGS) ./main.go $(ARGS)

docker-build: ## Build docker images with the KEDA Operator and Metrics Server.
DOCKER_BUILDKIT=1 docker build . -t ${IMAGE_CONTROLLER} --build-arg BUILD_VERSION=${VERSION} --build-arg GIT_VERSION=${GIT_VERSION} --build-arg GIT_COMMIT=${GIT_COMMIT}
DOCKER_BUILDKIT=1 docker build -f Dockerfile.adapter -t ${IMAGE_ADAPTER} . --build-arg BUILD_VERSION=${VERSION} --build-arg GIT_VERSION=${GIT_VERSION} --build-arg GIT_COMMIT=${GIT_COMMIT}
DOCKER_BUILDKIT=1 docker build -f Dockerfile.webhooks -t ${IMAGE_WEBHOOKS} . --build-arg BUILD_VERSION=${VERSION} --build-arg GIT_VERSION=${GIT_VERSION} --build-arg GIT_COMMIT=${GIT_COMMIT}

publish: docker-build ## Push images on to Container Registry (default: ghcr.io).
docker push $(IMAGE_CONTROLLER)
docker push $(IMAGE_ADAPTER)
docker push $(IMAGE_WEBHOOKS)

publish-controller-multiarch: ## Build and push multi-arch Docker image for KEDA Operator.
docker buildx build --output=type=${OUTPUT_TYPE} --platform=${BUILD_PLATFORMS} . -t ${IMAGE_CONTROLLER} --build-arg BUILD_VERSION=${VERSION} --build-arg GIT_VERSION=${GIT_VERSION} --build-arg GIT_COMMIT=${GIT_COMMIT}

publish-adapter-multiarch: ## Build and push multi-arch Docker image for KEDA Metrics Server.
docker buildx build --output=type=${OUTPUT_TYPE} --platform=${BUILD_PLATFORMS} -f Dockerfile.adapter -t ${IMAGE_ADAPTER} . --build-arg BUILD_VERSION=${VERSION} --build-arg GIT_VERSION=${GIT_VERSION} --build-arg GIT_COMMIT=${GIT_COMMIT}

publish-multiarch: publish-controller-multiarch publish-adapter-multiarch ## Push multi-arch Docker images on to Container Registry (default: ghcr.io).
publish-webhooks-multiarch: ## Build and push multi-arch Docker image for KEDA Hooks.
docker buildx build --output=type=${OUTPUT_TYPE} --platform=${BUILD_PLATFORMS} -f Dockerfile.webhooks -t ${IMAGE_WEBHOOKS} . --build-arg BUILD_VERSION=${VERSION} --build-arg GIT_VERSION=${GIT_VERSION} --build-arg GIT_COMMIT=${GIT_COMMIT}

publish-multiarch: publish-controller-multiarch publish-adapter-multiarch publish-webhooks-multiarch ## Push multi-arch Docker images on to Container Registry (default: ghcr.io).

release: manifests kustomize set-version ## Produce new KEDA release in keda-$(VERSION).yaml file.
cd config/manager && \
$(KUSTOMIZE) edit set image ghcr.io/kedacore/keda=${IMAGE_CONTROLLER}
cd config/metrics-server && \
$(KUSTOMIZE) edit set image ghcr.io/kedacore/keda-metrics-apiserver=${IMAGE_ADAPTER}
cd config/webhooks && \
$(KUSTOMIZE) edit set image ghcr.io/kedacore/keda-webhooks=${IMAGE_WEBHOOKS}
# Need this workaround to mitigate a problem with inserting labels into selectors,
# until this issue is solved: https://github.com/kubernetes-sigs/kustomize/issues/1009
@sed -i".out" -e 's@version:[ ].*@version: $(VERSION)@g' config/default/kustomize-config/metadataLabelTransformer.yaml
Expand All @@ -209,6 +222,7 @@ release: manifests kustomize set-version ## Produce new KEDA release in keda-$(V
sign-images: ## Sign KEDA images published on GitHub Container Registry
COSIGN_EXPERIMENTAL=1 cosign sign ${COSIGN_FLAGS} $(IMAGE_CONTROLLER)
COSIGN_EXPERIMENTAL=1 cosign sign ${COSIGN_FLAGS} $(IMAGE_ADAPTER)
COSIGN_EXPERIMENTAL=1 cosign sign ${COSIGN_FLAGS} $(IMAGE_WEBHOOKS)

.PHONY: set-version
set-version:
Expand Down Expand Up @@ -246,6 +260,9 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in
$(KUSTOMIZE) edit add annotation --force cloud.google.com/workload-identity-provider:${GCP_WI_PROVIDER} cloud.google.com/service-account-email:${TF_GCP_SA_EMAIL} cloud.google.com/gcloud-run-as-user:${NON_ROOT_USER_ID}; \
fi

cd config/webhooks && \
$(KUSTOMIZE) edit set image ghcr.io/kedacore/keda-webhooks=${IMAGE_WEBHOOKS}

# Need this workaround to mitigate a problem with inserting labels into selectors,
# until this issue is solved: https://github.com/kubernetes-sigs/kustomize/issues/1009
@sed -i".out" -e 's@version:[ ].*@version: $(VERSION)@g' config/default/kustomize-config/metadataLabelTransformer.yaml
Expand Down
11 changes: 7 additions & 4 deletions PROJECT
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,19 @@ resources:
domain: keda.sh
group: keda
kind: ScaledObject
path: github.com/kedacore/keda/v2/apis/keda/v1alpha1
path: github.com/kedacore/keda/apis/keda/v1alpha1
version: v1alpha1
webhooks:
validation: true
webhookVersion: v1
- api:
crdVersion: v1
namespaced: true
controller: true
domain: keda.sh
group: keda
kind: ScaledJob
path: github.com/kedacore/keda/v2/apis/keda/v1alpha1
path: github.com/kedacore/keda/apis/keda/v1alpha1
version: v1alpha1
- api:
crdVersion: v1
Expand All @@ -30,7 +33,7 @@ resources:
domain: keda.sh
group: keda
kind: TriggerAuthentication
path: github.com/kedacore/keda/v2/apis/keda/v1alpha1
path: github.com/kedacore/keda/apis/keda/v1alpha1
version: v1alpha1
- api:
crdVersion: v1
Expand All @@ -39,6 +42,6 @@ resources:
domain: keda.sh
group: keda
kind: ClusterTriggerAuthentication
path: github.com/kedacore/keda/v2/apis/keda/v1alpha1
path: github.com/kedacore/keda/apis/keda/v1alpha1
version: v1alpha1
version: "3"
112 changes: 112 additions & 0 deletions apis/keda/v1alpha1/scaledobject_webhook.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
/*
Copyright 2023 The KEDA Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
"context"
"encoding/json"
"fmt"

autoscalingv2 "k8s.io/api/autoscaling/v2"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/webhook"
)

var scaledobjectlog = logf.Log.WithName("scaledobject-validation-webhook")

var kc client.Client

func (r *ScaledObject) SetupWebhookWithManager(mgr ctrl.Manager) error {
kc = mgr.GetClient()
return ctrl.NewWebhookManagedBy(mgr).
For(r).
Complete()
}

// +kubebuilder:webhook:path=/validate-keda-sh-v1alpha1-scaledobject,mutating=false,failurePolicy=ignore,sideEffects=None,groups=keda.sh,resources=scaledobjects,verbs=create;update,versions=v1alpha1,name=vscaledobject.kb.io,admissionReviewVersions=v1
// +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=validatingwebhookconfigurations,verbs=get;list;watch;update;patch
// +kubebuilder:rbac:groups="",namespace=keda,resources=secrets,verbs=get;list;watch;create;update;patch;delete

var _ webhook.Validator = &ScaledObject{}

// ValidateCreate implements webhook.Validator so a webhook will be registered for the type
func (r *ScaledObject) ValidateCreate() error {
val, _ := json.MarshalIndent(r, "", " ")
scaledobjectlog.V(1).Info(fmt.Sprintf("validating scaledobject creation for %s", string(val)))
return validateWorkload(r)
}

func (r *ScaledObject) ValidateUpdate(old runtime.Object) error {
val, _ := json.MarshalIndent(r, "", " ")
scaledobjectlog.V(1).Info(fmt.Sprintf("validating scaledobject update for %s", string(val)))
return validateWorkload(r)
}

func validateWorkload(so *ScaledObject) error {
hpaList := &autoscalingv2.HorizontalPodAutoscalerList{}
opt := &client.ListOptions{
Namespace: so.Namespace,
}
err := kc.List(context.Background(), hpaList, opt)
if err != nil {
return err
}

for _, hpa := range hpaList.Items {
val, _ := json.MarshalIndent(hpa, "", " ")
scaledobjectlog.V(1).Info(fmt.Sprintf("checking hpa %s: %v", hpa.Name, string(val)))
hpaTarget := hpa.Spec.ScaleTargetRef
soTarget := so.Spec.ScaleTargetRef

hpaTargetAPI := "apps/v1"
if hpaTarget.APIVersion != "" {
hpaTargetAPI = hpaTarget.APIVersion
}
hpaTargetKind := "Deployment"
if hpaTarget.Kind != "" {
hpaTargetKind = hpaTarget.Kind
}
soTargetAPI := "apps/v1"
if soTarget.APIVersion != "" {
soTargetAPI = soTarget.APIVersion
}
soTargetKind := "Deployment"
if soTarget.Kind != "" {
soTargetKind = soTarget.Kind
}

if hpaTargetAPI == soTargetAPI &&
hpaTargetKind == soTargetKind &&
hpaTarget.Name == soTarget.Name {
labels := hpa.GetLabels()
if val, ok := labels[ScaledObjectOwnerAnnotation]; !ok || val != so.Name {
err := fmt.Errorf("the workload %s of type %s/%s is already managed by the hpa %s, and it isn't managed by this scaledobject", soTarget.Name, soTargetAPI, soTargetKind, hpa.Name)
scaledobjectlog.Error(err, "validation error")
return err
}
}
}
scaledobjectlog.V(1).Info(fmt.Sprintf("scaledobject %s is valid", so.Name))
return nil
}

func (r *ScaledObject) ValidateDelete() error {
return nil
}
Loading

0 comments on commit 7756767

Please sign in to comment.