From 434bcbcfa6fac178077ec404e3ef07c31fb912ad Mon Sep 17 00:00:00 2001 From: STRRL Date: Sat, 7 Sep 2024 21:21:43 -0700 Subject: [PATCH] feat: setup gatewayclass support Signed-off-by: STRRL --- .../main.go | 30 +++- hack/dev/deployment.yaml | 74 +--------- hack/dev/gatewayclass.yaml | 6 + hack/dev/rbac.yaml | 79 +++++++++++ .../templates/clusterrole.yaml | 13 +- .../templates/gatewayclass.yaml | 6 + pkg/controller/bootstrap.go | 13 +- pkg/controller/gatewayclass-controller.go | 133 ++++++++++++++++++ skaffold.yaml | 4 + test/integration/controller/suite_test.go | 8 +- 10 files changed, 289 insertions(+), 77 deletions(-) create mode 100644 hack/dev/gatewayclass.yaml create mode 100644 hack/dev/rbac.yaml create mode 100644 helm/cloudflare-tunnel-ingress-controller/templates/gatewayclass.yaml create mode 100644 pkg/controller/gatewayclass-controller.go diff --git a/cmd/cloudflare-tunnel-ingress-controller/main.go b/cmd/cloudflare-tunnel-ingress-controller/main.go index 750bdb1..fd0d405 100644 --- a/cmd/cloudflare-tunnel-ingress-controller/main.go +++ b/cmd/cloudflare-tunnel-ingress-controller/main.go @@ -2,18 +2,22 @@ package main import ( "context" + "log" + "os" + "time" + cloudflarecontroller "github.com/STRRL/cloudflare-tunnel-ingress-controller/pkg/cloudflare-controller" "github.com/STRRL/cloudflare-tunnel-ingress-controller/pkg/controller" "github.com/cloudflare/cloudflare-go" "github.com/go-logr/logr" "github.com/go-logr/stdr" "github.com/spf13/cobra" - "log" - "os" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client/config" crlog "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" - "time" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" ) type rootCmdFlags struct { @@ -72,7 +76,20 @@ func main() { os.Exit(1) } - mgr, err := manager.New(cfg, manager.Options{}) + scheme := runtime.NewScheme() + err = clientgoscheme.AddToScheme(scheme) + if err != nil { + logger.Error(err, "unable to add scheme") + os.Exit(1) + } + // append gateway-api scheme + err = gatewayv1.AddToScheme(scheme) + if err != nil { + logger.Error(err, "unable to add gateway-api scheme") + os.Exit(1) + } + + mgr, err := manager.New(cfg, manager.Options{Scheme: scheme}) if err != nil { logger.Error(err, "unable to set up manager") os.Exit(1) @@ -89,6 +106,11 @@ func main() { return err } + err = controller.RegisterGatewayClassController(logger, mgr) + if err != nil { + return err + } + ticker := time.NewTicker(10 * time.Second) done := make(chan struct{}) defer close(done) diff --git a/hack/dev/deployment.yaml b/hack/dev/deployment.yaml index 80d64a5..dffd6e2 100644 --- a/hack/dev/deployment.yaml +++ b/hack/dev/deployment.yaml @@ -11,74 +11,6 @@ spec: selector: app: cloudflare-tunnel-ingress-controller --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cloudflare-tunnel-ingress-controller - labels: - app: cloudflare-tunnel-ingress-controller -rules: - - apiGroups: - - "" - resources: - - services - - endpoints - - secrets - verbs: - - get - - list - - watch - - apiGroups: - - networking.k8s.io - resources: - - ingresses - - ingressclasses - verbs: - - get - - list - - watch - - update - - apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update - - apiGroups: - - apps - resources: - - deployments - verbs: - - get - - list - - watch - - update - - create ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cloudflare-tunnel-ingress-controller - namespace: cloudflare-tunnel-ingress-controller-dev - labels: - app: cloudflare-tunnel-ingress-controller ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cloudflare-tunnel-ingress-controller - labels: - app: cloudflare-tunnel-ingress-controller -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cloudflare-tunnel-ingress-controller -subjects: - - name: cloudflare-tunnel-ingress-controller - kind: ServiceAccount - # hardcoded namespace for dev - namespace: cloudflare-tunnel-ingress-controller-dev ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -132,4 +64,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: CLOUDFLARED_IMAGE + value: "cloudflare/cloudflared:latest" + - name: CLOUDFLARED_IMAGE_PULL_POLICY + value: "IfNotPresent" + - name: CLOUDFLARED_REPLICA_COUNT + value: "1" serviceAccountName: cloudflare-tunnel-ingress-controller diff --git a/hack/dev/gatewayclass.yaml b/hack/dev/gatewayclass.yaml new file mode 100644 index 0000000..d0c0b13 --- /dev/null +++ b/hack/dev/gatewayclass.yaml @@ -0,0 +1,6 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: cloudflare-tunnel +spec: + controllerName: "strrl.dev/cloudflare-tunnel-gatewayclass-controller" diff --git a/hack/dev/rbac.yaml b/hack/dev/rbac.yaml new file mode 100644 index 0000000..b57dff1 --- /dev/null +++ b/hack/dev/rbac.yaml @@ -0,0 +1,79 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cloudflare-tunnel-ingress-controller + labels: + app: cloudflare-tunnel-ingress-controller +rules: + - apiGroups: + - "" + resources: + - services + - endpoints + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + - ingressclasses + verbs: + - get + - list + - watch + - update + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - apps + resources: + - deployments + verbs: + - get + - list + - watch + - update + - create + - apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + - gatewayclasses/status + verbs: + - get + - list + - watch + - update + - patch +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloudflare-tunnel-ingress-controller + namespace: cloudflare-tunnel-ingress-controller-dev + labels: + app: cloudflare-tunnel-ingress-controller +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cloudflare-tunnel-ingress-controller + labels: + app: cloudflare-tunnel-ingress-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cloudflare-tunnel-ingress-controller +subjects: + - name: cloudflare-tunnel-ingress-controller + kind: ServiceAccount + # hardcoded namespace for dev + namespace: cloudflare-tunnel-ingress-controller-dev diff --git a/helm/cloudflare-tunnel-ingress-controller/templates/clusterrole.yaml b/helm/cloudflare-tunnel-ingress-controller/templates/clusterrole.yaml index e3ce1fc..b5b724e 100644 --- a/helm/cloudflare-tunnel-ingress-controller/templates/clusterrole.yaml +++ b/helm/cloudflare-tunnel-ingress-controller/templates/clusterrole.yaml @@ -38,4 +38,15 @@ rules: - list - watch - update - - create \ No newline at end of file + - create + - apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + - gatewayclasses/status + verbs: + - get + - list + - watch + - update + - patch diff --git a/helm/cloudflare-tunnel-ingress-controller/templates/gatewayclass.yaml b/helm/cloudflare-tunnel-ingress-controller/templates/gatewayclass.yaml new file mode 100644 index 0000000..d0c0b13 --- /dev/null +++ b/helm/cloudflare-tunnel-ingress-controller/templates/gatewayclass.yaml @@ -0,0 +1,6 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: cloudflare-tunnel +spec: + controllerName: "strrl.dev/cloudflare-tunnel-gatewayclass-controller" diff --git a/pkg/controller/bootstrap.go b/pkg/controller/bootstrap.go index 1dd69ee..8464d32 100644 --- a/pkg/controller/bootstrap.go +++ b/pkg/controller/bootstrap.go @@ -6,6 +6,7 @@ import ( networkingv1 "k8s.io/api/networking/v1" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/manager" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" ) type IngressControllerOptions struct { @@ -26,8 +27,18 @@ func RegisterIngressController(logger logr.Logger, mgr manager.Manager, options return err } + return nil +} + +func RegisterGatewayClassController(logger logr.Logger, mgr manager.Manager) error { + controller := NewGatewayClassController(logger.WithName("gatewayclass-controller"), mgr.GetClient()) + err := builder. + ControllerManagedBy(mgr). + For(&gatewayv1.GatewayClass{}). + Complete(controller) + if err != nil { - logger.WithName("register-controller").Error(err, "could not register ingress class controller") + logger.WithName("register-controller").Error(err, "could not register gatewayclass controller") return err } diff --git a/pkg/controller/gatewayclass-controller.go b/pkg/controller/gatewayclass-controller.go new file mode 100644 index 0000000..94abb14 --- /dev/null +++ b/pkg/controller/gatewayclass-controller.go @@ -0,0 +1,133 @@ +package controller + +import ( + "context" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +// GatewayClassController should implement the Reconciler interface +var _ reconcile.Reconciler = &GatewayClassController{} + +const ( + GatewayClassControllerFinalizer = "strrl.dev/cloudflare-tunnel-gatewayclass-controller-controlled" + ControllerName = "strrl.dev/cloudflare-tunnel-gatewayclass-controller" +) + +type GatewayClassController struct { + logger logr.Logger + kubeClient client.Client +} + +func NewGatewayClassController(logger logr.Logger, kubeClient client.Client) *GatewayClassController { + return &GatewayClassController{logger: logger, kubeClient: kubeClient} +} + +func (g *GatewayClassController) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + origin := gatewayv1.GatewayClass{} + err := g.kubeClient.Get(ctx, request.NamespacedName, &origin) + if err != nil { + if apierrors.IsNotFound(err) { + return reconcile.Result{}, nil + } + return reconcile.Result{}, errors.Wrapf(err, "fetch gatewayclass %s", request.NamespacedName) + } + + controlled, err := g.isControlledByThisController(ctx, origin) + if err != nil && !apierrors.IsNotFound(err) { + return reconcile.Result{}, errors.Wrapf(err, "check if gatewayclass %s is controlled by this controller", request.NamespacedName) + } + + if !controlled { + g.logger.V(1).Info("gatewayclass is NOT controlled by this controller", + "gatewayclass", request.NamespacedName, + "controlled-controller-name", ControllerName, + ) + return reconcile.Result{ + Requeue: false, + }, nil + } + + g.logger.V(1).Info("gatewayclass is controlled by this controller", + "gatewayclass", request.NamespacedName, + "controlled-controller-name", ControllerName, + ) + + // Update GatewayClass status + if err := g.updateGatewayClassStatus(ctx, &origin); err != nil { + return reconcile.Result{}, errors.Wrapf(err, "update status for gatewayclass %s", request.NamespacedName) + } + + g.logger.Info("update cloudflare tunnel config", "triggered-by", request.NamespacedName) + + err = g.attachFinalizer(ctx, *(origin.DeepCopy())) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "attach finalizer to gatewayclass %s", request.NamespacedName) + } + + // Add your custom logic here + + if origin.DeletionTimestamp != nil { + err = g.cleanFinalizer(ctx, origin) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "clean finalizer from gatewayclass %s", request.NamespacedName) + } + } + + g.logger.V(3).Info("reconcile completed", "triggered-by", request.NamespacedName) + return reconcile.Result{}, nil +} + +func (g *GatewayClassController) isControlledByThisController(ctx context.Context, target gatewayv1.GatewayClass) (bool, error) { + if string(target.Spec.ControllerName) == ControllerName { + return true, nil + } + return false, nil +} + +func (g *GatewayClassController) attachFinalizer(ctx context.Context, gatewayClass gatewayv1.GatewayClass) error { + if stringSliceContains(gatewayClass.Finalizers, GatewayClassControllerFinalizer) { + return nil + } + gatewayClass.Finalizers = append(gatewayClass.Finalizers, GatewayClassControllerFinalizer) + err := g.kubeClient.Update(ctx, &gatewayClass) + if err != nil { + return errors.Wrapf(err, "attach finalizer for %s", gatewayClass.Name) + } + return nil +} + +func (g *GatewayClassController) cleanFinalizer(ctx context.Context, gatewayClass gatewayv1.GatewayClass) error { + if !stringSliceContains(gatewayClass.Finalizers, GatewayClassControllerFinalizer) { + return nil + } + gatewayClass.Finalizers = removeStringFromSlice(gatewayClass.Finalizers, GatewayClassControllerFinalizer) + err := g.kubeClient.Update(ctx, &gatewayClass) + if err != nil { + return errors.Wrapf(err, "clean finalizer for %s", gatewayClass.Name) + } + return nil +} + +// Update the updateGatewayClassStatus method +func (g *GatewayClassController) updateGatewayClassStatus(ctx context.Context, gatewayClass *gatewayv1.GatewayClass) error { + newCondition := metav1.Condition{ + Type: string(gatewayv1.GatewayClassConditionStatusAccepted), + Status: metav1.ConditionTrue, + ObservedGeneration: gatewayClass.Generation, + LastTransitionTime: metav1.Now(), + Reason: string(gatewayv1.GatewayClassReasonAccepted), + Message: "GatewayClass has been accepted by the controller", + } + + meta.SetStatusCondition(&gatewayClass.Status.Conditions, newCondition) + + return g.kubeClient.Status().Update(ctx, gatewayClass) +} diff --git a/skaffold.yaml b/skaffold.yaml index 6f5ba8b..03c1fec 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -17,4 +17,8 @@ manifests: - hack/dev/ns.yaml - hack/dev/cloudflare-api.yaml - hack/dev/deployment.yaml + - hack/dev/gatewayclass.yaml - hack/dev/ingress-class.yaml + - hack/dev/rbac.yaml # Add this line +deploy: + kubectl: {} diff --git a/test/integration/controller/suite_test.go b/test/integration/controller/suite_test.go index c73ec76..acc2a1f 100644 --- a/test/integration/controller/suite_test.go +++ b/test/integration/controller/suite_test.go @@ -2,18 +2,19 @@ package controller import ( "context" + "log" + "os" + "testing" + "github.com/go-logr/stdr" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" - "log" - "os" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" - "testing" ) var ( @@ -46,6 +47,7 @@ var _ = BeforeSuite(func() { scheme := runtime.NewScheme() err = clientgoscheme.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) kubeClient, err = client.New(cfg, client.Options{Scheme: scheme})