From 22870faa5ad4e1aeb25497e34e7a83a602cd2f66 Mon Sep 17 00:00:00 2001 From: october <764213885@qq.com> Date: Fri, 9 Sep 2022 00:56:01 +0800 Subject: [PATCH 1/7] feat(gateway): impl gateway crd and opt route&domain crd --- controllers/gateway/api/v1/domain_types.go | 35 +- controllers/gateway/api/v1/gateway_types.go | 52 +-- controllers/gateway/api/v1/route_types.go | 8 + .../gateway/controllers/gateway_controller.go | 298 +++++++++++++++++- .../gateway/controllers/route_controller.go | 35 +- 5 files changed, 375 insertions(+), 53 deletions(-) diff --git a/controllers/gateway/api/v1/domain_types.go b/controllers/gateway/api/v1/domain_types.go index 7034076278..2ff1c9e1b8 100644 --- a/controllers/gateway/api/v1/domain_types.go +++ b/controllers/gateway/api/v1/domain_types.go @@ -26,28 +26,39 @@ import ( // DomainSpec defines the desired state of Domain type DomainSpec struct { - // Preferred 是提供域名使用位置的推荐项,是字符串类型,长度大于1小于10,必须存在 - // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:MaxLength=10 + // Domain是域名,必须存在,匹配域名规则 + // +kubebuilder:validation:Pattern="^([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$" // +kubebuilder:validation:Required - Preferred string `json:"preferred"` + Domain string `json:"domain"` - // Region 是域名设定的解析区域,是字符串类型,长度大于1小于10,可选存在 - // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:MaxLength=10 - // +kubebuilder:validation:Optional - Region string `json:"region,omitempty"` + // BackendType是后端服务类型,必须存在APP;OSS;WEBSITE + // +kubebuilder:validation:Enum=APP;OSS;WEBSITE + // +kubebuilder:validation:Required + BackendType BackendType `json:"backendType"` - // Domain 是域名,是字符串类型,规则匹配域名规则,必须存在 - // +kubebuilder:validation:Pattern="^([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$" + // Region 是区域 必须存在 由字符数组和-组成 + // +kubebuilder:validation:Pattern="^[a-zA-Z0-9-]+$" // +kubebuilder:validation:Required - Domain string `json:"domain"` + Region string `json:"region"` + + // Cluster 是网关集群配置 必须存在 + // +kubebuilder:validation:Required + Cluster ClusterSpec `json:"cluster"` // CertConfigRef 是字符串类型,是configMap的引用 // +kubebuilder:validation:Required CertConfigRef string `json:"certConfigRef"` } +// ClusterSpec 是集群的规格 +type ClusterSpec struct { + // url是集群的url,必须存在 + Url string `json:"url"` + + // key是集群的key,必须存在 + Key string `json:"key"` +} + // DomainStatus defines the observed state of Domain type DomainStatus struct { // CertConfigRef 是字符串类型,是configMap的引用 diff --git a/controllers/gateway/api/v1/gateway_types.go b/controllers/gateway/api/v1/gateway_types.go index 512d974c20..2c05969e97 100644 --- a/controllers/gateway/api/v1/gateway_types.go +++ b/controllers/gateway/api/v1/gateway_types.go @@ -23,6 +23,23 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. +// BackendType 是后端服务类型 +type BackendType string + +const ( + APP BackendType = "app" + BUCKET BackendType = "bucket" + WEBSITE BackendType = "website" +) + +// RouteState 是路由的状态 +type RouteState string + +const ( + PREPARING RouteState = "preparing" + CREATED RouteState = "created" +) + // GatewaySpec defines the desired state of Gateway type GatewaySpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster @@ -49,37 +66,26 @@ type GatewayStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster // Important: Run "make" to regenerate code after modifying this file - // AppDomain 是应用域名,必须存在 - // +kubebuilder:validation:Required - AppDomain string `json:"appDomain"` + // AppRoute 是应用路由 + AppRoute *GatewayRoute `json:"appRoute,omitempty"` - // BucketDomains 是存储桶域名列表,是一个数组,可选存在 - // +kubebuilder:validation:Optional - BucketDomains []string `json:"bucketDomains,omitempty"` + // BucketRoutes 是存储桶路由 + BucketRoutes map[string]*GatewayRoute `json:"bucketRoutes,omitempty"` - // WebsiteDomains 是静态站点域名列表,是一个数组,可选存在 - // +kubebuilder:validation:Optional - WebsiteDomains []string `json:"websiteDomains,omitempty"` + // WebsiteRoutes 是静态站点路由 + WebsiteRoutes map[string]*GatewayRoute `json:"websiteRoutes,omitempty"` } -// BucketDomain 是存储桶位的域名配置 -type BucketDomain struct { - // Name 是存储桶名称,必须存在 - // +kubebuilder:validation:Required - Name string `json:"name"` - - // Domain 是存储桶域名,必须存在 +type GatewayRoute struct { + // DomainName 是域名名称,必须存在 // +kubebuilder:validation:Required - Domain string `json:"domain"` -} + DomainName string `json:"domainName"` -// WebsiteDomain 是静态站点的域名配置 -type WebsiteDomain struct { - // Name 是静态站点名称,必须存在 + // DomainNamespace 是域名所在的命名空间,必须存在 // +kubebuilder:validation:Required - Name string `json:"name"` + DomainNamespace string `json:"domainNamespace"` - // Domain 是静态站点域名,必须存在 + // Domain 是域名,必须存在 // +kubebuilder:validation:Required Domain string `json:"domain"` } diff --git a/controllers/gateway/api/v1/route_types.go b/controllers/gateway/api/v1/route_types.go index 85bc3fec27..70b10c5b5b 100644 --- a/controllers/gateway/api/v1/route_types.go +++ b/controllers/gateway/api/v1/route_types.go @@ -36,6 +36,14 @@ type RouteSpec struct { // +kubebuilder:validation:Required Backend Backend `json:"backend"` + // DomainName 是域名名称,必须存在 + // +kubebuilder:validation:Required + DomainName string `json:"domainName"` + + // DomainNamespace 是域名所在的命名空间,必须存在 + // +kubebuilder:validation:Required + DomainNamespace string `json:"domainNamespace"` + // CertConfigRef 是证书配置,可选存在 // +kubebuilder:validation:Optional CertConfigRef string `json:"certConfigRef,omitempty"` diff --git a/controllers/gateway/controllers/gateway_controller.go b/controllers/gateway/controllers/gateway_controller.go index fee0b29412..ea2eb32b80 100644 --- a/controllers/gateway/controllers/gateway_controller.go +++ b/controllers/gateway/controllers/gateway_controller.go @@ -18,6 +18,8 @@ package controllers import ( "context" + ossv1 "github.com/labring/laf/controllers/oss/api/v1" + "laf/pkg/util" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" @@ -48,12 +50,306 @@ type GatewayReconciler struct { // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/reconcile func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { _ = log.FromContext(ctx) + // get route + gateway := &gatewayv1.Gateway{} + if err := r.Get(ctx, req.NamespacedName, gateway); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + if !gateway.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } - // TODO(user): your logic here + return r.apply(ctx, gateway) +} + +func (r *GatewayReconciler) delete(ctx context.Context, gateway *gatewayv1.Gateway) (ctrl.Result, error) { + + // delete app route + if gateway.Status.AppRoute != nil { + if _, err := r.deleteApp(ctx, gateway); err != nil { + return ctrl.Result{}, err + } + } + + // delete bucket route + if gateway.Status.BucketRoutes != nil { + if _, err := r.deleteBucket(ctx, gateway); err != nil { + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, nil +} + +func (r *GatewayReconciler) apply(ctx context.Context, gateway *gatewayv1.Gateway) (ctrl.Result, error) { + if gateway.Status.AppRoute == nil { + result, err := r.applyApp(ctx, gateway) + if err != nil { + return result, err + } + } + + // 添加bucket 路由 + if gateway.Status.BucketRoutes == nil || len(gateway.Status.BucketRoutes) > len(gateway.Status.BucketRoutes) { + result, err := r.applyBucket(ctx, gateway) + if err != nil { + return result, err + } + } + + return ctrl.Result{}, nil +} + +// applyDomain apply domain +func (r *GatewayReconciler) applyApp(ctx context.Context, gateway *gatewayv1.Gateway) (ctrl.Result, error) { + _log := log.FromContext(ctx) + + finalizerName := "app.gateway.laf.dev" + + // add finalizer + if !util.ContainsString(gateway.GetFinalizers(), finalizerName) { + gateway.SetFinalizers(append(gateway.GetFinalizers(), finalizerName)) + if err := r.Update(ctx, gateway); err != nil { + return ctrl.Result{}, err + } + _log.Info("added finalizer", "finalizer", finalizerName) + } + + // TODO select app from application + region := "default" + + // select app domain + appDomain, err := r.selectDomain(ctx, gatewayv1.APP, region) + if err != nil { + return ctrl.Result{}, err + } + if appDomain == nil { + _log.Info("no app domain found") + } + + // set gateway status + gateway.Status.AppRoute = &gatewayv1.GatewayRoute{ + DomainName: appDomain.Name, + DomainNamespace: appDomain.Namespace, + Domain: gateway.Spec.AppId + "." + appDomain.Spec.Domain, + } + + // create app route + appRoute := &gatewayv1.Route{ + ObjectMeta: ctrl.ObjectMeta{ + Name: "app", + Namespace: gateway.Spec.AppId, + }, + Spec: gatewayv1.RouteSpec{ + Domain: gateway.Status.AppRoute.Domain, + DomainName: appDomain.Name, + DomainNamespace: appDomain.Namespace, + Backend: gatewayv1.Backend{ + ServiceName: gateway.Spec.AppId, + ServicePort: 80, + }, + }, + } + if err := r.Create(ctx, appRoute); err != nil { + return ctrl.Result{}, err + } + + // update gateway status + if err := r.Status().Update(ctx, gateway); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil +} + +// deleteApp delete app +func (r *GatewayReconciler) deleteApp(ctx context.Context, gateway *gatewayv1.Gateway) (ctrl.Result, error) { + _log := log.FromContext(ctx) + + finalizerName := "app.gateway.laf.dev" + + // delete app route + appRoute := &gatewayv1.Route{} + if err := r.Get(ctx, client.ObjectKey{Name: "app", Namespace: gateway.Spec.AppId}, appRoute); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + if err := r.Delete(ctx, appRoute); err != nil { + return ctrl.Result{}, err + } + + // remove finalizer + if util.ContainsString(gateway.GetFinalizers(), finalizerName) { + gateway.SetFinalizers(util.RemoveString(gateway.GetFinalizers(), finalizerName)) + if err := r.Update(ctx, gateway); err != nil { + return ctrl.Result{}, err + } + _log.Info("removed finalizer", "finalizer", finalizerName) + } + + gateway.Status.AppRoute = nil + if err := r.Status().Update(ctx, gateway); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{}, nil } +// applyBucket apply bucket +func (r *GatewayReconciler) applyBucket(ctx context.Context, gateway *gatewayv1.Gateway) (ctrl.Result, error) { + _log := log.FromContext(ctx) + + // if gateway status bucketRoutes is nil, init it + if gateway.Status.BucketRoutes == nil { + gateway.Status.BucketRoutes = make(map[string]*gatewayv1.GatewayRoute, 0) + } + + // select bucket domain + for _, bucketName := range gateway.Spec.Buckets { + + finalizerName := bucketName + ".bucket.gateway.laf.dev" + + // add finalizer + if !util.ContainsString(gateway.GetFinalizers(), finalizerName) { + gateway.SetFinalizers(append(gateway.GetFinalizers(), finalizerName)) + if err := r.Update(ctx, gateway); err != nil { + return ctrl.Result{}, err + } + _log.Info("added finalizer", "finalizer", finalizerName) + } + + // if bucket route is not exist, create it + if _, ok := gateway.Status.BucketRoutes[bucketName]; ok { + continue + } + + // get bucket + bucket := ossv1.Bucket{} + err := r.Get(ctx, client.ObjectKey{Namespace: gateway.Namespace, Name: bucketName}, &bucket) + if err != nil { + return ctrl.Result{}, err + } + // get user + user := ossv1.User{} + err = r.Get(ctx, client.ObjectKey{Namespace: gateway.Namespace, Name: bucket.Status.User}, &user) + if err != nil { + return ctrl.Result{}, err + } + // get store + store := ossv1.Store{} + err = r.Get(ctx, client.ObjectKey{Namespace: gateway.Namespace, Name: user.Status.StoreName}, &store) + if err != nil { + return ctrl.Result{}, err + } + // select bucket domain + bucketDomain, err := r.selectDomain(ctx, gatewayv1.BUCKET, store.Spec.Region) + if err != nil { + return ctrl.Result{}, err + } + if bucketDomain == nil { + _log.Info("no bucket domain found") + continue + } + + gateway.Status.BucketRoutes[bucketName] = &gatewayv1.GatewayRoute{ + DomainName: bucketDomain.Name, + DomainNamespace: bucketDomain.Namespace, + Domain: bucketName + "." + bucketDomain.Spec.Domain, + } + + // create bucket route + bucketRoute := &gatewayv1.Route{ + ObjectMeta: ctrl.ObjectMeta{ + Name: "bucket-" + bucketName, + Namespace: gateway.Spec.AppId, + }, + Spec: gatewayv1.RouteSpec{ + Domain: gateway.Status.BucketRoutes[bucketName].Domain, + DomainName: bucketDomain.Name, + DomainNamespace: bucketDomain.Namespace, + Backend: gatewayv1.Backend{ + ServiceName: user.Status.Endpoint, + ServicePort: 0, // If set to 0, the port is not used + }, + PassHost: bucketName + "." + user.Status.Endpoint, + }, + } + if err := r.Create(ctx, bucketRoute); err != nil { + return ctrl.Result{}, err + } + + // update gateway status + if err := r.Status().Update(ctx, gateway); err != nil { + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, nil +} + +// deleteBucket delete bucket +func (r *GatewayReconciler) deleteBucket(ctx context.Context, gateway *gatewayv1.Gateway) (ctrl.Result, error) { + _ = log.FromContext(ctx) + + // convert gateway buckets to map + bucketMap := make(map[string]bool, 0) + for _, bucketName := range gateway.Spec.Buckets { + bucketMap[bucketName] = true + } + + // find deleted bucket, remote route and finalizer + for bucketName, _ := range gateway.Status.BucketRoutes { + if _, ok := bucketMap[bucketName]; !ok { + // delete route + route := &gatewayv1.Route{} + err := r.Get(ctx, client.ObjectKey{Namespace: gateway.Spec.AppId, Name: "bucket-" + bucketName}, route) + if err != nil { + return ctrl.Result{}, err + } + if err := r.Delete(ctx, route); err != nil { + return ctrl.Result{}, err + } + // delete finalizer + finalizerName := bucketName + ".bucket.gateway.laf.dev" + gateway.SetFinalizers(util.RemoveString(gateway.GetFinalizers(), finalizerName)) + if err := r.Update(ctx, gateway); err != nil { + return ctrl.Result{}, err + } + + // delete bucket route + delete(gateway.Status.BucketRoutes, bucketName) + } + } + + gateway.Status.BucketRoutes = nil + if err := r.Status().Update(ctx, gateway); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +func (r *GatewayReconciler) selectDomain(ctx context.Context, backendType gatewayv1.BackendType, region string) (*gatewayv1.Domain, error) { + _ = log.FromContext(ctx) + + // get all domains + var domains gatewayv1.DomainList + if err := r.List(ctx, &domains); err != nil { + return nil, err + } + + // select domain + for _, domain := range domains.Items { + if domain.Spec.BackendType != backendType { + continue + } + + if domain.Spec.Region != region { + continue + } + return &domain, nil + } + return nil, nil +} + // SetupWithManager sets up the controller with the Manager. func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). diff --git a/controllers/gateway/controllers/route_controller.go b/controllers/gateway/controllers/route_controller.go index bd277e4362..d3f30ab26a 100644 --- a/controllers/gateway/controllers/route_controller.go +++ b/controllers/gateway/controllers/route_controller.go @@ -19,10 +19,11 @@ package controllers import ( "context" "github.com/labring/laf/controllers/gateway/apisix" + "k8s.io/apimachinery/pkg/types" "laf/pkg/util" + "strconv" "time" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -56,9 +57,9 @@ func (r *RouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl _ = log.FromContext(ctx) // 获取apiSix集群操作对象 - apiSixCluster, result, err := r.getApiSixCluster(ctx, &gatewayv1.Route{}) + apiSixCluster, err := r.getGatewayCluster(ctx, &gatewayv1.Route{}) if err != nil { - return result, err + return ctrl.Result{}, err } // get the route @@ -95,10 +96,16 @@ func (r *RouteReconciler) applyRoute(ctx context.Context, route *gatewayv1.Route "uri": "/*", "host": route.Spec.Domain, } + + // if port is not 0, set node to serviceName and servicePort combination + node := route.Spec.Backend.ServiceName + if route.Spec.Backend.ServicePort != 0 { + node += ":" + strconv.FormatInt(int64(route.Spec.Backend.ServicePort), 10) + } upstream := map[string]interface{}{ "type": "roundrobin", "nodes": map[string]interface{}{ - route.Spec.Domain: 1, + node: 1, }, } @@ -163,20 +170,14 @@ func (r *RouteReconciler) deleteRoute(ctx context.Context, route *gatewayv1.Rout return ctrl.Result{}, nil } -func (r *RouteReconciler) getApiSixCluster(ctx context.Context, route *gatewayv1.Route) (*apisix.Cluster, ctrl.Result, error) { - // 这块是否应该从全局配置拿apisix地址和adminKey ? - // 还是直接根据习惯拿默认的apisix service地址和adminKey - //cluster := apisix.NewCluster("http://apisix-admin.apisix.svc.cluster.local:9180/apisix/admin/", "edd1c9f034335f136f87ad84b625c8f1") - cluster := apisix.NewCluster("http://localhost:29180/apisix/admin/", "edd1c9f034335f136f87ad84b625c8f1") - return cluster, ctrl.Result{}, nil -} - -func (r *RouteReconciler) getGlobalConfig(ctx context.Context, config *corev1.ConfigMap) (ctrl.Result, error) { - err := r.Get(ctx, client.ObjectKey{Namespace: "laf-cloud", Name: "laf-config"}, config) - if err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) +func (r *RouteReconciler) getGatewayCluster(ctx context.Context, route *gatewayv1.Route) (*apisix.Cluster, error) { + // get domain + domain := gatewayv1.Domain{} + if err := r.Get(ctx, types.NamespacedName{Name: route.Spec.Domain}, &domain); err != nil { + return nil, err } - return ctrl.Result{}, nil + cluster := apisix.NewCluster(domain.Spec.Cluster.Url, domain.Spec.Cluster.Key) + return cluster, nil } func getRouteId(route *gatewayv1.Route) string { From 8ee4cad26de7896f1444a5f14bd382ea0b646a5b Mon Sep 17 00:00:00 2001 From: october <764213885@qq.com> Date: Fri, 9 Sep 2022 01:00:47 +0800 Subject: [PATCH 2/7] feat(gateway):update manifests --- .../crd/bases/gateway.laf.dev_domains.yaml | 36 +++++++--- .../crd/bases/gateway.laf.dev_gateways.yaml | 70 +++++++++++++++---- .../crd/bases/gateway.laf.dev_routes.yaml | 8 +++ 3 files changed, 89 insertions(+), 25 deletions(-) diff --git a/controllers/gateway/config/crd/bases/gateway.laf.dev_domains.yaml b/controllers/gateway/config/crd/bases/gateway.laf.dev_domains.yaml index 3a215e6a0e..5405150bfb 100644 --- a/controllers/gateway/config/crd/bases/gateway.laf.dev_domains.yaml +++ b/controllers/gateway/config/crd/bases/gateway.laf.dev_domains.yaml @@ -35,27 +35,43 @@ spec: spec: description: DomainSpec defines the desired state of Domain properties: + backendType: + description: BackendType是后端服务类型,必须存在APP;OSS;WEBSITE + enum: + - APP + - OSS + - WEBSITE + type: string certConfigRef: description: CertConfigRef 是字符串类型,是configMap的引用 type: string + cluster: + description: Cluster 是网关集群配置 必须存在 + properties: + key: + description: key是集群的key,必须存在 + type: string + url: + description: url是集群的url,必须存在 + type: string + required: + - key + - url + type: object domain: - description: Domain 是域名,是字符串类型,规则匹配域名规则,必须存在 + description: Domain是域名,必须存在,匹配域名规则 pattern: ^([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6}$ type: string - preferred: - description: Preferred 是提供域名使用位置的推荐项,是字符串类型,长度大于1小于10,必须存在 - maxLength: 10 - minLength: 1 - type: string region: - description: Region 是域名设定的解析区域,是字符串类型,长度大于1小于10,可选存在 - maxLength: 10 - minLength: 1 + description: Region 是区域 必须存在 由字符数组和-组成 + pattern: ^[a-zA-Z0-9-]+$ type: string required: + - backendType - certConfigRef + - cluster - domain - - preferred + - region type: object status: description: DomainStatus defines the observed state of Domain diff --git a/controllers/gateway/config/crd/bases/gateway.laf.dev_gateways.yaml b/controllers/gateway/config/crd/bases/gateway.laf.dev_gateways.yaml index 557828d3ab..1a082fda5f 100644 --- a/controllers/gateway/config/crd/bases/gateway.laf.dev_gateways.yaml +++ b/controllers/gateway/config/crd/bases/gateway.laf.dev_gateways.yaml @@ -55,21 +55,61 @@ spec: status: description: GatewayStatus defines the observed state of Gateway properties: - appDomain: - description: AppDomain 是应用域名,必须存在 - type: string - bucketDomains: - description: BucketDomains 是存储桶域名列表,是一个数组,可选存在 - items: - type: string - type: array - websiteDomains: - description: WebsiteDomains 是静态站点域名列表,是一个数组,可选存在 - items: - type: string - type: array - required: - - appDomain + appRoute: + description: AppRoute 是应用路由 + properties: + domain: + description: Domain 是域名,必须存在 + type: string + domainName: + description: DomainName 是域名名称,必须存在 + type: string + domainNamespace: + description: DomainNamespace 是域名所在的命名空间,必须存在 + type: string + required: + - domain + - domainName + - domainNamespace + type: object + bucketRoutes: + additionalProperties: + properties: + domain: + description: Domain 是域名,必须存在 + type: string + domainName: + description: DomainName 是域名名称,必须存在 + type: string + domainNamespace: + description: DomainNamespace 是域名所在的命名空间,必须存在 + type: string + required: + - domain + - domainName + - domainNamespace + type: object + description: BucketRoutes 是存储桶路由 + type: object + websiteRoutes: + additionalProperties: + properties: + domain: + description: Domain 是域名,必须存在 + type: string + domainName: + description: DomainName 是域名名称,必须存在 + type: string + domainNamespace: + description: DomainNamespace 是域名所在的命名空间,必须存在 + type: string + required: + - domain + - domainName + - domainNamespace + type: object + description: WebsiteRoutes 是静态站点路由 + type: object type: object type: object served: true diff --git a/controllers/gateway/config/crd/bases/gateway.laf.dev_routes.yaml b/controllers/gateway/config/crd/bases/gateway.laf.dev_routes.yaml index 51477880f4..7d9f9cbd0f 100644 --- a/controllers/gateway/config/crd/bases/gateway.laf.dev_routes.yaml +++ b/controllers/gateway/config/crd/bases/gateway.laf.dev_routes.yaml @@ -55,6 +55,12 @@ spec: domain: description: Domain 是路由域名,必须存在 type: string + domainName: + description: DomainName 是域名名称,必须存在 + type: string + domainNamespace: + description: DomainNamespace 是域名所在的命名空间,必须存在 + type: string enableWebSocket: description: EnableWebSocket 是否开启websocket, 默认否 type: boolean @@ -77,6 +83,8 @@ spec: required: - backend - domain + - domainName + - domainNamespace type: object status: description: RouteStatus defines the observed state of Route From d2170f67b13a0ecf204a86c807c687760e72fc64 Mon Sep 17 00:00:00 2001 From: october <764213885@qq.com> Date: Mon, 12 Sep 2022 23:14:01 +0800 Subject: [PATCH 3/7] feat(gateway): update app route --- controllers/gateway/api/v1/domain_types.go | 6 +- controllers/gateway/api/v1/gateway_types.go | 2 +- .../gateway/api/v1/zz_generated.deepcopy.go | 87 +++++++++++++------ controllers/gateway/apisix/route.go | 2 +- controllers/gateway/apisix/route_test.go | 29 ------- .../crd/bases/gateway.laf.dev_domains.yaml | 9 +- .../crd/bases/gateway.laf.dev_gateways.yaml | 4 +- .../config/samples/gateway_v1_domain.yaml | 7 +- .../config/samples/gateway_v1_gateway.yaml | 2 +- .../gateway/controllers/gateway_controller.go | 14 +-- .../gateway/controllers/route_controller.go | 14 +-- 11 files changed, 86 insertions(+), 90 deletions(-) diff --git a/controllers/gateway/api/v1/domain_types.go b/controllers/gateway/api/v1/domain_types.go index 2ff1c9e1b8..499337a00c 100644 --- a/controllers/gateway/api/v1/domain_types.go +++ b/controllers/gateway/api/v1/domain_types.go @@ -32,7 +32,7 @@ type DomainSpec struct { Domain string `json:"domain"` // BackendType是后端服务类型,必须存在APP;OSS;WEBSITE - // +kubebuilder:validation:Enum=APP;OSS;WEBSITE + // +kubebuilder:validation:Enum=app;oss;website // +kubebuilder:validation:Required BackendType BackendType `json:"backendType"` @@ -45,8 +45,8 @@ type DomainSpec struct { // +kubebuilder:validation:Required Cluster ClusterSpec `json:"cluster"` - // CertConfigRef 是字符串类型,是configMap的引用 - // +kubebuilder:validation:Required + // CertConfigRef 是字符串类型,是configMap的引用,可选存在 + // +kubebuilder:validation:Optional CertConfigRef string `json:"certConfigRef"` } diff --git a/controllers/gateway/api/v1/gateway_types.go b/controllers/gateway/api/v1/gateway_types.go index 2c05969e97..a574632282 100644 --- a/controllers/gateway/api/v1/gateway_types.go +++ b/controllers/gateway/api/v1/gateway_types.go @@ -50,7 +50,7 @@ type GatewaySpec struct { // AppId是应用id,字母数字组成,长度5至16位,必须存在 // +kubebuilder:validation:Pattern="^[a-zA-Z0-9]{5,16}$" // +kubebuilder:validation:Required - AppId string `json:"appId"` + AppId string `json:"appid"` // Buckets是存储桶, 是一个数组,可选存在 // +kubebuilder:validation:Optional diff --git a/controllers/gateway/api/v1/zz_generated.deepcopy.go b/controllers/gateway/api/v1/zz_generated.deepcopy.go index f8fe67ced1..dee0e1db50 100644 --- a/controllers/gateway/api/v1/zz_generated.deepcopy.go +++ b/controllers/gateway/api/v1/zz_generated.deepcopy.go @@ -41,16 +41,16 @@ func (in *Backend) DeepCopy() *Backend { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BucketDomain) DeepCopyInto(out *BucketDomain) { +func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = *in } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BucketDomain. -func (in *BucketDomain) DeepCopy() *BucketDomain { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSpec. +func (in *ClusterSpec) DeepCopy() *ClusterSpec { if in == nil { return nil } - out := new(BucketDomain) + out := new(ClusterSpec) in.DeepCopyInto(out) return out } @@ -117,6 +117,7 @@ func (in *DomainList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DomainSpec) DeepCopyInto(out *DomainSpec) { *out = *in + out.Cluster = in.Cluster } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DomainSpec. @@ -203,6 +204,21 @@ func (in *GatewayList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GatewayRoute) DeepCopyInto(out *GatewayRoute) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayRoute. +func (in *GatewayRoute) DeepCopy() *GatewayRoute { + if in == nil { + return nil + } + out := new(GatewayRoute) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GatewaySpec) DeepCopyInto(out *GatewaySpec) { *out = *in @@ -231,15 +247,40 @@ func (in *GatewaySpec) DeepCopy() *GatewaySpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GatewayStatus) DeepCopyInto(out *GatewayStatus) { *out = *in - if in.BucketDomains != nil { - in, out := &in.BucketDomains, &out.BucketDomains - *out = make([]string, len(*in)) - copy(*out, *in) + if in.AppRoute != nil { + in, out := &in.AppRoute, &out.AppRoute + *out = new(GatewayRoute) + **out = **in } - if in.WebsiteDomains != nil { - in, out := &in.WebsiteDomains, &out.WebsiteDomains - *out = make([]string, len(*in)) - copy(*out, *in) + if in.BucketRoutes != nil { + in, out := &in.BucketRoutes, &out.BucketRoutes + *out = make(map[string]*GatewayRoute, len(*in)) + for key, val := range *in { + var outVal *GatewayRoute + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = new(GatewayRoute) + **out = **in + } + (*out)[key] = outVal + } + } + if in.WebsiteRoutes != nil { + in, out := &in.WebsiteRoutes, &out.WebsiteRoutes + *out = make(map[string]*GatewayRoute, len(*in)) + for key, val := range *in { + var outVal *GatewayRoute + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = new(GatewayRoute) + **out = **in + } + (*out)[key] = outVal + } } } @@ -273,7 +314,7 @@ func (in *Route) DeepCopyInto(out *Route) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) out.Status = in.Status } @@ -331,6 +372,11 @@ func (in *RouteList) DeepCopyObject() runtime.Object { func (in *RouteSpec) DeepCopyInto(out *RouteSpec) { *out = *in out.Backend = in.Backend + if in.PathRewrite != nil { + in, out := &in.PathRewrite, &out.PathRewrite + *out = new(PathRewrite) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RouteSpec. @@ -357,18 +403,3 @@ func (in *RouteStatus) DeepCopy() *RouteStatus { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *WebsiteDomain) DeepCopyInto(out *WebsiteDomain) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebsiteDomain. -func (in *WebsiteDomain) DeepCopy() *WebsiteDomain { - if in == nil { - return nil - } - out := new(WebsiteDomain) - in.DeepCopyInto(out) - return out -} diff --git a/controllers/gateway/apisix/route.go b/controllers/gateway/apisix/route.go index dcd80bc0a0..90cc5ee4aa 100644 --- a/controllers/gateway/apisix/route.go +++ b/controllers/gateway/apisix/route.go @@ -16,6 +16,6 @@ func (r *RouteClient) Put(id string, data map[string]interface{}) error { // Delete 删除路由 func (r *RouteClient) Delete(id string) error { - url := r.client.baseURL + "routes/" + id + url := r.client.baseURL + "/routes/" + id return r.client.Delete(url, "routes") } diff --git a/controllers/gateway/apisix/route_test.go b/controllers/gateway/apisix/route_test.go index bbf597a68a..f4a77cfa1d 100644 --- a/controllers/gateway/apisix/route_test.go +++ b/controllers/gateway/apisix/route_test.go @@ -1,30 +1 @@ package apisix - -import "testing" - -func TestCreateRoute(t *testing.T) { - // 创建RouteClient - routeClient := NewRouteClient("http://localhost:9180", "edd1c9f034335f136f87ad84b625c8f1") - routeData := map[string]interface{}{ - "uri": "/hello", - "upstream": map[string]interface{}{ - "type": "roundrobin", - "nodes": map[string]interface{}{ - "baidu.com": 1, - }, - }, - "host": "test.com", - } - err := routeClient.PutRoute("test", routeData) - if err != nil { - panic(err) - } -} - -func TestDeleteRoute(t *testing.T) { - routeClient := NewRouteClient("http://localhost:9180", "edd1c9f034335f136f87ad84b625c8f1") - err := routeClient.DeleteRoute("test") - if err != nil { - panic(err) - } -} diff --git a/controllers/gateway/config/crd/bases/gateway.laf.dev_domains.yaml b/controllers/gateway/config/crd/bases/gateway.laf.dev_domains.yaml index 5405150bfb..110ec9a850 100644 --- a/controllers/gateway/config/crd/bases/gateway.laf.dev_domains.yaml +++ b/controllers/gateway/config/crd/bases/gateway.laf.dev_domains.yaml @@ -38,12 +38,12 @@ spec: backendType: description: BackendType是后端服务类型,必须存在APP;OSS;WEBSITE enum: - - APP - - OSS - - WEBSITE + - app + - oss + - website type: string certConfigRef: - description: CertConfigRef 是字符串类型,是configMap的引用 + description: CertConfigRef 是字符串类型,是configMap的引用,可选存在 type: string cluster: description: Cluster 是网关集群配置 必须存在 @@ -68,7 +68,6 @@ spec: type: string required: - backendType - - certConfigRef - cluster - domain - region diff --git a/controllers/gateway/config/crd/bases/gateway.laf.dev_gateways.yaml b/controllers/gateway/config/crd/bases/gateway.laf.dev_gateways.yaml index 1a082fda5f..3a0c5ffe33 100644 --- a/controllers/gateway/config/crd/bases/gateway.laf.dev_gateways.yaml +++ b/controllers/gateway/config/crd/bases/gateway.laf.dev_gateways.yaml @@ -35,7 +35,7 @@ spec: spec: description: GatewaySpec defines the desired state of Gateway properties: - appId: + appid: description: AppId是应用id,字母数字组成,长度5至16位,必须存在 pattern: ^[a-zA-Z0-9]{5,16}$ type: string @@ -50,7 +50,7 @@ spec: type: string type: array required: - - appId + - appid type: object status: description: GatewayStatus defines the observed state of Gateway diff --git a/controllers/gateway/config/samples/gateway_v1_domain.yaml b/controllers/gateway/config/samples/gateway_v1_domain.yaml index dae1008fe4..43d235cfaa 100644 --- a/controllers/gateway/config/samples/gateway_v1_domain.yaml +++ b/controllers/gateway/config/samples/gateway_v1_domain.yaml @@ -3,4 +3,9 @@ kind: Domain metadata: name: domain-sample spec: - # TODO(user): Add fields here + domain: app.laf.win + backendType: app + region: default + cluster: + url: http://localhost:19180/apisix/admin + key: edd1c9f034335f136f87ad84b625c8f1 \ No newline at end of file diff --git a/controllers/gateway/config/samples/gateway_v1_gateway.yaml b/controllers/gateway/config/samples/gateway_v1_gateway.yaml index eff821424d..f76182f939 100644 --- a/controllers/gateway/config/samples/gateway_v1_gateway.yaml +++ b/controllers/gateway/config/samples/gateway_v1_gateway.yaml @@ -3,4 +3,4 @@ kind: Gateway metadata: name: gateway-sample spec: - # TODO(user): Add fields here + appid: default diff --git a/controllers/gateway/controllers/gateway_controller.go b/controllers/gateway/controllers/gateway_controller.go index ea2eb32b80..4ba211aa8a 100644 --- a/controllers/gateway/controllers/gateway_controller.go +++ b/controllers/gateway/controllers/gateway_controller.go @@ -56,7 +56,7 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, client.IgnoreNotFound(err) } if !gateway.DeletionTimestamp.IsZero() { - return ctrl.Result{}, nil + return r.delete(ctx, gateway) } return r.apply(ctx, gateway) @@ -170,7 +170,7 @@ func (r *GatewayReconciler) deleteApp(ctx context.Context, gateway *gatewayv1.Ga // delete app route appRoute := &gatewayv1.Route{} if err := r.Get(ctx, client.ObjectKey{Name: "app", Namespace: gateway.Spec.AppId}, appRoute); err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) + return ctrl.Result{}, err } if err := r.Delete(ctx, appRoute); err != nil { return ctrl.Result{}, err @@ -185,11 +185,6 @@ func (r *GatewayReconciler) deleteApp(ctx context.Context, gateway *gatewayv1.Ga _log.Info("removed finalizer", "finalizer", finalizerName) } - gateway.Status.AppRoute = nil - if err := r.Status().Update(ctx, gateway); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{}, nil } @@ -319,11 +314,6 @@ func (r *GatewayReconciler) deleteBucket(ctx context.Context, gateway *gatewayv1 } } - gateway.Status.BucketRoutes = nil - if err := r.Status().Update(ctx, gateway); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{}, nil } diff --git a/controllers/gateway/controllers/route_controller.go b/controllers/gateway/controllers/route_controller.go index d3f30ab26a..02feb20dd8 100644 --- a/controllers/gateway/controllers/route_controller.go +++ b/controllers/gateway/controllers/route_controller.go @@ -56,18 +56,18 @@ type RouteReconciler struct { func (r *RouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { _ = log.FromContext(ctx) - // 获取apiSix集群操作对象 - apiSixCluster, err := r.getGatewayCluster(ctx, &gatewayv1.Route{}) - if err != nil { - return ctrl.Result{}, err - } - // get the route var route gatewayv1.Route if err := r.Get(ctx, req.NamespacedName, &route); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } + // 获取apiSix集群操作对象 + apiSixCluster, err := r.getGatewayCluster(ctx, &route) + if err != nil { + return ctrl.Result{}, err + } + // 如果路由已经被设置删除标记,则删除路由 if !route.ObjectMeta.DeletionTimestamp.IsZero() { return r.deleteRoute(ctx, &route, apiSixCluster) @@ -173,7 +173,7 @@ func (r *RouteReconciler) deleteRoute(ctx context.Context, route *gatewayv1.Rout func (r *RouteReconciler) getGatewayCluster(ctx context.Context, route *gatewayv1.Route) (*apisix.Cluster, error) { // get domain domain := gatewayv1.Domain{} - if err := r.Get(ctx, types.NamespacedName{Name: route.Spec.Domain}, &domain); err != nil { + if err := r.Get(ctx, types.NamespacedName{Namespace: route.Spec.DomainNamespace, Name: route.Spec.DomainName}, &domain); err != nil { return nil, err } cluster := apisix.NewCluster(domain.Spec.Cluster.Url, domain.Spec.Cluster.Key) From 89134fb6d5635e4921db66e3eec3d0b5ef33b1dc Mon Sep 17 00:00:00 2001 From: october <764213885@qq.com> Date: Tue, 13 Sep 2022 20:00:05 +0800 Subject: [PATCH 4/7] feat(gateway): update bucket route --- controllers/gateway/api/v1/domain_types.go | 4 ++-- .../gateway/config/crd/bases/gateway.laf.dev_domains.yaml | 4 ++-- controllers/gateway/controllers/gateway_controller.go | 2 +- controllers/gateway/main.go | 3 +++ 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/controllers/gateway/api/v1/domain_types.go b/controllers/gateway/api/v1/domain_types.go index 499337a00c..b469c7b06b 100644 --- a/controllers/gateway/api/v1/domain_types.go +++ b/controllers/gateway/api/v1/domain_types.go @@ -31,8 +31,8 @@ type DomainSpec struct { // +kubebuilder:validation:Required Domain string `json:"domain"` - // BackendType是后端服务类型,必须存在APP;OSS;WEBSITE - // +kubebuilder:validation:Enum=app;oss;website + // BackendType是后端服务类型,必须存在APP;bucket;WEBSITE + // +kubebuilder:validation:Enum=app;bucket;website // +kubebuilder:validation:Required BackendType BackendType `json:"backendType"` diff --git a/controllers/gateway/config/crd/bases/gateway.laf.dev_domains.yaml b/controllers/gateway/config/crd/bases/gateway.laf.dev_domains.yaml index 110ec9a850..6be417dc6b 100644 --- a/controllers/gateway/config/crd/bases/gateway.laf.dev_domains.yaml +++ b/controllers/gateway/config/crd/bases/gateway.laf.dev_domains.yaml @@ -36,10 +36,10 @@ spec: description: DomainSpec defines the desired state of Domain properties: backendType: - description: BackendType是后端服务类型,必须存在APP;OSS;WEBSITE + description: BackendType是后端服务类型,必须存在APP;bucket;WEBSITE enum: - app - - oss + - bucket - website type: string certConfigRef: diff --git a/controllers/gateway/controllers/gateway_controller.go b/controllers/gateway/controllers/gateway_controller.go index 4ba211aa8a..8ba47c6100 100644 --- a/controllers/gateway/controllers/gateway_controller.go +++ b/controllers/gateway/controllers/gateway_controller.go @@ -247,7 +247,7 @@ func (r *GatewayReconciler) applyBucket(ctx context.Context, gateway *gatewayv1. gateway.Status.BucketRoutes[bucketName] = &gatewayv1.GatewayRoute{ DomainName: bucketDomain.Name, DomainNamespace: bucketDomain.Namespace, - Domain: bucketName + "." + bucketDomain.Spec.Domain, + Domain: gateway.Spec.AppId + "-" + bucketName + "." + bucketDomain.Spec.Domain, } // create bucket route diff --git a/controllers/gateway/main.go b/controllers/gateway/main.go index 0d3c81921c..2d29bf2ce0 100644 --- a/controllers/gateway/main.go +++ b/controllers/gateway/main.go @@ -18,6 +18,7 @@ package main import ( "flag" + ossv1 "github.com/labring/laf/controllers/oss/api/v1" "os" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -46,6 +47,8 @@ func init() { utilruntime.Must(gatewayv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme + + utilruntime.Must(ossv1.AddToScheme(scheme)) } func main() { From a40dfc78e31cacf960fce274e1653256e379d5a8 Mon Sep 17 00:00:00 2001 From: october <764213885@qq.com> Date: Wed, 14 Sep 2022 22:29:19 +0800 Subject: [PATCH 5/7] feat(gateway): opt controller --- .../gateway/controllers/gateway_controller.go | 194 +++++++++++------- .../gateway/controllers/route_controller.go | 8 +- 2 files changed, 126 insertions(+), 76 deletions(-) diff --git a/controllers/gateway/controllers/gateway_controller.go b/controllers/gateway/controllers/gateway_controller.go index 8ba47c6100..b5bdecc556 100644 --- a/controllers/gateway/controllers/gateway_controller.go +++ b/controllers/gateway/controllers/gateway_controller.go @@ -18,9 +18,11 @@ package controllers import ( "context" + "errors" ossv1 "github.com/labring/laf/controllers/oss/api/v1" "laf/pkg/util" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -29,6 +31,8 @@ import ( gatewayv1 "github.com/labring/laf/controllers/gateway/api/v1" ) +const gatewayFinalizer = "gateway.gateway.laf.dev" + // GatewayReconciler reconciles a Gateway object type GatewayReconciler struct { client.Client @@ -50,7 +54,7 @@ type GatewayReconciler struct { // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/reconcile func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { _ = log.FromContext(ctx) - // get route + // get gateway gateway := &gatewayv1.Gateway{} if err := r.Get(ctx, req.NamespacedName, gateway); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) @@ -63,25 +67,42 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } func (r *GatewayReconciler) delete(ctx context.Context, gateway *gatewayv1.Gateway) (ctrl.Result, error) { + _log := log.FromContext(ctx) // delete app route - if gateway.Status.AppRoute != nil { - if _, err := r.deleteApp(ctx, gateway); err != nil { - return ctrl.Result{}, err - } + if _, err := r.deleteApp(ctx, gateway); err != nil { + return ctrl.Result{}, err } // delete bucket route - if gateway.Status.BucketRoutes != nil { - if _, err := r.deleteBucket(ctx, gateway); err != nil { - return ctrl.Result{}, err - } + if _, err := r.deleteBuckets(ctx, gateway, nil); err != nil { + return ctrl.Result{}, err } + // remove finalizer + gateway.Finalizers = util.RemoveString(gateway.Finalizers, gatewayFinalizer) + if err := r.Update(ctx, gateway); err != nil { + return ctrl.Result{}, err + } + + _log.Info("delete gateway: v success", "v", gateway.Name) + return ctrl.Result{}, nil } func (r *GatewayReconciler) apply(ctx context.Context, gateway *gatewayv1.Gateway) (ctrl.Result, error) { + _log := log.FromContext(ctx) + + _log.Info("apply gateway", "v", gateway.Name) + + // add finalizer + if !util.ContainsString(gateway.Finalizers, gatewayFinalizer) { + gateway.Finalizers = append(gateway.Finalizers, gatewayFinalizer) + if err := r.Update(ctx, gateway); err != nil { + return ctrl.Result{}, err + } + } + if gateway.Status.AppRoute == nil { result, err := r.applyApp(ctx, gateway) if err != nil { @@ -89,12 +110,10 @@ func (r *GatewayReconciler) apply(ctx context.Context, gateway *gatewayv1.Gatewa } } - // 添加bucket 路由 - if gateway.Status.BucketRoutes == nil || len(gateway.Status.BucketRoutes) > len(gateway.Status.BucketRoutes) { - result, err := r.applyBucket(ctx, gateway) - if err != nil { - return result, err - } + // apply bucket route + result, err := r.applyBucket(ctx, gateway) + if err != nil { + return result, err } return ctrl.Result{}, nil @@ -104,17 +123,6 @@ func (r *GatewayReconciler) apply(ctx context.Context, gateway *gatewayv1.Gatewa func (r *GatewayReconciler) applyApp(ctx context.Context, gateway *gatewayv1.Gateway) (ctrl.Result, error) { _log := log.FromContext(ctx) - finalizerName := "app.gateway.laf.dev" - - // add finalizer - if !util.ContainsString(gateway.GetFinalizers(), finalizerName) { - gateway.SetFinalizers(append(gateway.GetFinalizers(), finalizerName)) - if err := r.Update(ctx, gateway); err != nil { - return ctrl.Result{}, err - } - _log.Info("added finalizer", "finalizer", finalizerName) - } - // TODO select app from application region := "default" @@ -125,10 +133,11 @@ func (r *GatewayReconciler) applyApp(ctx context.Context, gateway *gatewayv1.Gat } if appDomain == nil { _log.Info("no app domain found") + return ctrl.Result{}, errors.New("no app domain found") } // set gateway status - gateway.Status.AppRoute = &gatewayv1.GatewayRoute{ + routeStatus := &gatewayv1.GatewayRoute{ DomainName: appDomain.Name, DomainNamespace: appDomain.Namespace, Domain: gateway.Spec.AppId + "." + appDomain.Spec.Domain, @@ -141,7 +150,7 @@ func (r *GatewayReconciler) applyApp(ctx context.Context, gateway *gatewayv1.Gat Namespace: gateway.Spec.AppId, }, Spec: gatewayv1.RouteSpec{ - Domain: gateway.Status.AppRoute.Domain, + Domain: routeStatus.Domain, DomainName: appDomain.Name, DomainNamespace: appDomain.Namespace, Backend: gatewayv1.Backend{ @@ -151,10 +160,14 @@ func (r *GatewayReconciler) applyApp(ctx context.Context, gateway *gatewayv1.Gat }, } if err := r.Create(ctx, appRoute); err != nil { + if !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, nil + } return ctrl.Result{}, err } // update gateway status + gateway.Status.AppRoute = routeStatus if err := r.Status().Update(ctx, gateway); err != nil { return ctrl.Result{}, err } @@ -165,31 +178,63 @@ func (r *GatewayReconciler) applyApp(ctx context.Context, gateway *gatewayv1.Gat func (r *GatewayReconciler) deleteApp(ctx context.Context, gateway *gatewayv1.Gateway) (ctrl.Result, error) { _log := log.FromContext(ctx) - finalizerName := "app.gateway.laf.dev" + _log.Info("delete app route", "v", gateway.Name) // delete app route appRoute := &gatewayv1.Route{} + if err := r.Get(ctx, client.ObjectKey{Name: "app", Namespace: gateway.Spec.AppId}, appRoute); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } return ctrl.Result{}, err } + if err := r.Delete(ctx, appRoute); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } return ctrl.Result{}, err } - // remove finalizer - if util.ContainsString(gateway.GetFinalizers(), finalizerName) { - gateway.SetFinalizers(util.RemoveString(gateway.GetFinalizers(), finalizerName)) - if err := r.Update(ctx, gateway); err != nil { - return ctrl.Result{}, err + return ctrl.Result{}, nil +} + +// applyBucket apply bucket +func (r *GatewayReconciler) applyBucket(ctx context.Context, gateway *gatewayv1.Gateway) (ctrl.Result, error) { + + // create bucket route + createBuckets := make([]string, 0) + for _, bucketName := range gateway.Spec.Buckets { + if _, ok := gateway.Status.BucketRoutes[bucketName]; !ok { + createBuckets = append(createBuckets, bucketName) + } + } + if len(createBuckets) != 0 { + result, err := r.addBuckets(ctx, gateway, createBuckets) + if err != nil { + return result, err + } + } + + // delete bucket route + deleteBuckets := make([]string, 0) + for bucketName := range gateway.Status.BucketRoutes { + if !util.ContainsString(gateway.Spec.Buckets, bucketName) { + deleteBuckets = append(deleteBuckets, bucketName) + } + } + if len(deleteBuckets) != 0 { + result, err := r.deleteBuckets(ctx, gateway, deleteBuckets) + if err != nil { + return result, err } - _log.Info("removed finalizer", "finalizer", finalizerName) } return ctrl.Result{}, nil } -// applyBucket apply bucket -func (r *GatewayReconciler) applyBucket(ctx context.Context, gateway *gatewayv1.Gateway) (ctrl.Result, error) { +func (r *GatewayReconciler) addBuckets(ctx context.Context, gateway *gatewayv1.Gateway, buckets []string) (ctrl.Result, error) { _log := log.FromContext(ctx) // if gateway status bucketRoutes is nil, init it @@ -198,18 +243,7 @@ func (r *GatewayReconciler) applyBucket(ctx context.Context, gateway *gatewayv1. } // select bucket domain - for _, bucketName := range gateway.Spec.Buckets { - - finalizerName := bucketName + ".bucket.gateway.laf.dev" - - // add finalizer - if !util.ContainsString(gateway.GetFinalizers(), finalizerName) { - gateway.SetFinalizers(append(gateway.GetFinalizers(), finalizerName)) - if err := r.Update(ctx, gateway); err != nil { - return ctrl.Result{}, err - } - _log.Info("added finalizer", "finalizer", finalizerName) - } + for _, bucketName := range buckets { // if bucket route is not exist, create it if _, ok := gateway.Status.BucketRoutes[bucketName]; ok { @@ -244,7 +278,7 @@ func (r *GatewayReconciler) applyBucket(ctx context.Context, gateway *gatewayv1. continue } - gateway.Status.BucketRoutes[bucketName] = &gatewayv1.GatewayRoute{ + routeStatus := &gatewayv1.GatewayRoute{ DomainName: bucketDomain.Name, DomainNamespace: bucketDomain.Namespace, Domain: gateway.Spec.AppId + "-" + bucketName + "." + bucketDomain.Spec.Domain, @@ -257,7 +291,7 @@ func (r *GatewayReconciler) applyBucket(ctx context.Context, gateway *gatewayv1. Namespace: gateway.Spec.AppId, }, Spec: gatewayv1.RouteSpec{ - Domain: gateway.Status.BucketRoutes[bucketName].Domain, + Domain: routeStatus.Domain, DomainName: bucketDomain.Name, DomainNamespace: bucketDomain.Namespace, Backend: gatewayv1.Backend{ @@ -268,9 +302,13 @@ func (r *GatewayReconciler) applyBucket(ctx context.Context, gateway *gatewayv1. }, } if err := r.Create(ctx, bucketRoute); err != nil { + if apierrors.IsAlreadyExists(err) { + continue + } return ctrl.Result{}, err } + gateway.Status.BucketRoutes[bucketName] = routeStatus // update gateway status if err := r.Status().Update(ctx, gateway); err != nil { return ctrl.Result{}, err @@ -281,36 +319,46 @@ func (r *GatewayReconciler) applyBucket(ctx context.Context, gateway *gatewayv1. } // deleteBucket delete bucket -func (r *GatewayReconciler) deleteBucket(ctx context.Context, gateway *gatewayv1.Gateway) (ctrl.Result, error) { +func (r *GatewayReconciler) deleteBuckets(ctx context.Context, gateway *gatewayv1.Gateway, buckets []string) (ctrl.Result, error) { _ = log.FromContext(ctx) - // convert gateway buckets to map - bucketMap := make(map[string]bool, 0) - for _, bucketName := range gateway.Spec.Buckets { - bucketMap[bucketName] = true + // if buckets is nil, add all buckets + if buckets == nil { + buckets = make([]string, 0) + for bucketName := range gateway.Status.BucketRoutes { + buckets = append(buckets, bucketName) + } } // find deleted bucket, remote route and finalizer - for bucketName, _ := range gateway.Status.BucketRoutes { - if _, ok := bucketMap[bucketName]; !ok { - // delete route - route := &gatewayv1.Route{} - err := r.Get(ctx, client.ObjectKey{Namespace: gateway.Spec.AppId, Name: "bucket-" + bucketName}, route) - if err != nil { - return ctrl.Result{}, err - } - if err := r.Delete(ctx, route); err != nil { - return ctrl.Result{}, err + for _, bucketName := range buckets { + // delete route + route := &gatewayv1.Route{} + + // 删除名称为test的route + err := r.Get(ctx, client.ObjectKey{Namespace: gateway.Spec.AppId, Name: "bucket-" + bucketName}, route) + if err != nil { + + } + + if err := r.Get(ctx, client.ObjectKey{Namespace: gateway.Spec.AppId, Name: "bucket-" + bucketName}, route); err != nil { + if apierrors.IsNotFound(err) { + continue } - // delete finalizer - finalizerName := bucketName + ".bucket.gateway.laf.dev" - gateway.SetFinalizers(util.RemoveString(gateway.GetFinalizers(), finalizerName)) - if err := r.Update(ctx, gateway); err != nil { - return ctrl.Result{}, err + return ctrl.Result{}, err + } + + if err := r.Delete(ctx, route); err != nil { + if apierrors.IsNotFound(err) { + continue } + return ctrl.Result{}, err + } - // delete bucket route - delete(gateway.Status.BucketRoutes, bucketName) + // delete bucket route + delete(gateway.Status.BucketRoutes, bucketName) + if err := r.Status().Update(ctx, gateway); err != nil { + return ctrl.Result{}, err } } diff --git a/controllers/gateway/controllers/route_controller.go b/controllers/gateway/controllers/route_controller.go index 02feb20dd8..b9f3df8b95 100644 --- a/controllers/gateway/controllers/route_controller.go +++ b/controllers/gateway/controllers/route_controller.go @@ -143,9 +143,11 @@ func (r *RouteReconciler) applyRoute(ctx context.Context, route *gatewayv1.Route } // update route status - route.Status.Domain = route.Spec.Domain - if err := r.Status().Update(ctx, route); err != nil { - return ctrl.Result{}, err + if route.Status.Domain == "" { + route.Status.Domain = route.Spec.Domain + if err := r.Status().Update(ctx, route); err != nil { + return ctrl.Result{}, err + } } _log.Info("route applied: " + routeId) From 9b3fa4967de3450c7b29405a2cc8675140023a61 Mon Sep 17 00:00:00 2001 From: october <764213885@qq.com> Date: Thu, 15 Sep 2022 22:55:22 +0800 Subject: [PATCH 6/7] feat(gateway): opt controller & remove redundant code --- .../gateway/controllers/gateway_controller.go | 129 ++++++------------ .../gateway/controllers/route_controller.go | 36 +++-- 2 files changed, 69 insertions(+), 96 deletions(-) diff --git a/controllers/gateway/controllers/gateway_controller.go b/controllers/gateway/controllers/gateway_controller.go index b5bdecc556..de45288796 100644 --- a/controllers/gateway/controllers/gateway_controller.go +++ b/controllers/gateway/controllers/gateway_controller.go @@ -20,7 +20,10 @@ import ( "context" "errors" ossv1 "github.com/labring/laf/controllers/oss/api/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" "laf/pkg/util" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -59,50 +62,15 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct if err := r.Get(ctx, req.NamespacedName, gateway); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } - if !gateway.DeletionTimestamp.IsZero() { - return r.delete(ctx, gateway) + if gateway.DeletionTimestamp.IsZero() { + return r.apply(ctx, gateway) } - - return r.apply(ctx, gateway) -} - -func (r *GatewayReconciler) delete(ctx context.Context, gateway *gatewayv1.Gateway) (ctrl.Result, error) { - _log := log.FromContext(ctx) - - // delete app route - if _, err := r.deleteApp(ctx, gateway); err != nil { - return ctrl.Result{}, err - } - - // delete bucket route - if _, err := r.deleteBuckets(ctx, gateway, nil); err != nil { - return ctrl.Result{}, err - } - - // remove finalizer - gateway.Finalizers = util.RemoveString(gateway.Finalizers, gatewayFinalizer) - if err := r.Update(ctx, gateway); err != nil { - return ctrl.Result{}, err - } - - _log.Info("delete gateway: v success", "v", gateway.Name) - return ctrl.Result{}, nil } func (r *GatewayReconciler) apply(ctx context.Context, gateway *gatewayv1.Gateway) (ctrl.Result, error) { _log := log.FromContext(ctx) - _log.Info("apply gateway", "v", gateway.Name) - - // add finalizer - if !util.ContainsString(gateway.Finalizers, gatewayFinalizer) { - gateway.Finalizers = append(gateway.Finalizers, gatewayFinalizer) - if err := r.Update(ctx, gateway); err != nil { - return ctrl.Result{}, err - } - } - if gateway.Status.AppRoute == nil { result, err := r.applyApp(ctx, gateway) if err != nil { @@ -116,6 +84,8 @@ func (r *GatewayReconciler) apply(ctx context.Context, gateway *gatewayv1.Gatewa return result, err } + _log.Info("apply gateway: name success", "name", gateway.Name) + return ctrl.Result{}, nil } @@ -159,44 +129,22 @@ func (r *GatewayReconciler) applyApp(ctx context.Context, gateway *gatewayv1.Gat }, }, } - if err := r.Create(ctx, appRoute); err != nil { - if !apierrors.IsAlreadyExists(err) { - return ctrl.Result{}, nil + + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, appRoute, func() error { + if err := controllerutil.SetControllerReference(gateway, appRoute, r.Scheme); err != nil { + return err } + return nil + }); err != nil { return ctrl.Result{}, err } // update gateway status gateway.Status.AppRoute = routeStatus - if err := r.Status().Update(ctx, gateway); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{}, nil -} -// deleteApp delete app -func (r *GatewayReconciler) deleteApp(ctx context.Context, gateway *gatewayv1.Gateway) (ctrl.Result, error) { - _log := log.FromContext(ctx) - - _log.Info("delete app route", "v", gateway.Name) - - // delete app route - appRoute := &gatewayv1.Route{} - - if err := r.Get(ctx, client.ObjectKey{Name: "app", Namespace: gateway.Spec.AppId}, appRoute); err != nil { - if apierrors.IsNotFound(err) { - return ctrl.Result{}, nil - } + if err = r.updateStatus(ctx, types.NamespacedName{Name: gateway.Name, Namespace: gateway.Namespace}, gateway.Status.DeepCopy()); err != nil { return ctrl.Result{}, err } - - if err := r.Delete(ctx, appRoute); err != nil { - if apierrors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - return ctrl.Result{}, nil } @@ -234,6 +182,7 @@ func (r *GatewayReconciler) applyBucket(ctx context.Context, gateway *gatewayv1. return ctrl.Result{}, nil } +// addBuckets add buckets func (r *GatewayReconciler) addBuckets(ctx context.Context, gateway *gatewayv1.Gateway, buckets []string) (ctrl.Result, error) { _log := log.FromContext(ctx) @@ -301,16 +250,18 @@ func (r *GatewayReconciler) addBuckets(ctx context.Context, gateway *gatewayv1.G PassHost: bucketName + "." + user.Status.Endpoint, }, } - if err := r.Create(ctx, bucketRoute); err != nil { - if apierrors.IsAlreadyExists(err) { - continue + + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, bucketRoute, func() error { + if err := controllerutil.SetControllerReference(gateway, bucketRoute, r.Scheme); err != nil { + return err } + return nil + }); err != nil { return ctrl.Result{}, err } - gateway.Status.BucketRoutes[bucketName] = routeStatus - // update gateway status - if err := r.Status().Update(ctx, gateway); err != nil { + + if err = r.updateStatus(ctx, types.NamespacedName{Name: gateway.Name, Namespace: gateway.Namespace}, gateway.Status.DeepCopy()); err != nil { return ctrl.Result{}, err } } @@ -318,29 +269,16 @@ func (r *GatewayReconciler) addBuckets(ctx context.Context, gateway *gatewayv1.G return ctrl.Result{}, nil } -// deleteBucket delete bucket +// deleteBuckets delete buckets func (r *GatewayReconciler) deleteBuckets(ctx context.Context, gateway *gatewayv1.Gateway, buckets []string) (ctrl.Result, error) { _ = log.FromContext(ctx) - // if buckets is nil, add all buckets - if buckets == nil { - buckets = make([]string, 0) - for bucketName := range gateway.Status.BucketRoutes { - buckets = append(buckets, bucketName) - } - } - // find deleted bucket, remote route and finalizer for _, bucketName := range buckets { // delete route route := &gatewayv1.Route{} // 删除名称为test的route - err := r.Get(ctx, client.ObjectKey{Namespace: gateway.Spec.AppId, Name: "bucket-" + bucketName}, route) - if err != nil { - - } - if err := r.Get(ctx, client.ObjectKey{Namespace: gateway.Spec.AppId, Name: "bucket-" + bucketName}, route); err != nil { if apierrors.IsNotFound(err) { continue @@ -357,7 +295,7 @@ func (r *GatewayReconciler) deleteBuckets(ctx context.Context, gateway *gatewayv // delete bucket route delete(gateway.Status.BucketRoutes, bucketName) - if err := r.Status().Update(ctx, gateway); err != nil { + if err := r.updateStatus(ctx, types.NamespacedName{Name: gateway.Name, Namespace: gateway.Namespace}, gateway.Status.DeepCopy()); err != nil { return ctrl.Result{}, err } } @@ -388,6 +326,23 @@ func (r *GatewayReconciler) selectDomain(ctx context.Context, backendType gatewa return nil, nil } +func (r *GatewayReconciler) updateStatus(ctx context.Context, nn types.NamespacedName, status *gatewayv1.GatewayStatus) error { + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + original := &gatewayv1.Gateway{} + if err := r.Get(ctx, nn, original); err != nil { + return err + } + original.Status = *status + if err := r.Client.Status().Update(ctx, original); err != nil { + return err + } + return nil + }); err != nil { + return err + } + return nil +} + // SetupWithManager sets up the controller with the Manager. func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). diff --git a/controllers/gateway/controllers/route_controller.go b/controllers/gateway/controllers/route_controller.go index b9f3df8b95..16e1e95004 100644 --- a/controllers/gateway/controllers/route_controller.go +++ b/controllers/gateway/controllers/route_controller.go @@ -20,7 +20,8 @@ import ( "context" "github.com/labring/laf/controllers/gateway/apisix" "k8s.io/apimachinery/pkg/types" - "laf/pkg/util" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "strconv" "time" @@ -80,12 +81,10 @@ func (r *RouteReconciler) applyRoute(ctx context.Context, route *gatewayv1.Route _log := log.FromContext(ctx) // add finalizer if not present - if !util.ContainsString(route.ObjectMeta.Finalizers, routeFinalizer) { - route.ObjectMeta.Finalizers = append(route.ObjectMeta.Finalizers, routeFinalizer) + if controllerutil.AddFinalizer(route, routeFinalizer) { if err := r.Update(ctx, route); err != nil { return ctrl.Result{}, err } - _log.Info("add finalizer to route") } // get route id @@ -145,7 +144,7 @@ func (r *RouteReconciler) applyRoute(ctx context.Context, route *gatewayv1.Route // update route status if route.Status.Domain == "" { route.Status.Domain = route.Spec.Domain - if err := r.Status().Update(ctx, route); err != nil { + if err := r.updateStatus(ctx, types.NamespacedName{Namespace: route.Namespace, Name: route.Name}, route.Status.DeepCopy()); err != nil { return ctrl.Result{}, err } } @@ -160,14 +159,16 @@ func (r *RouteReconciler) deleteRoute(ctx context.Context, route *gatewayv1.Rout routeId := getRouteId(route) err := cluster.Route.Delete(routeId) if err != nil { - return ctrl.Result{RequeueAfter: time.Minute * 15}, err + return ctrl.Result{RequeueAfter: time.Minute * 1}, err } // remove the finalizer - route.ObjectMeta.Finalizers = util.RemoveString(route.ObjectMeta.Finalizers, routeFinalizer) - if err := r.Update(ctx, route); err != nil { - return ctrl.Result{}, err + if controllerutil.RemoveFinalizer(route, routeFinalizer) { + if err := r.Update(ctx, route); err != nil { + return ctrl.Result{}, err + } } + _log.Info("route deleted: " + routeId) return ctrl.Result{}, nil } @@ -182,6 +183,23 @@ func (r *RouteReconciler) getGatewayCluster(ctx context.Context, route *gatewayv return cluster, nil } +func (r *RouteReconciler) updateStatus(ctx context.Context, nn types.NamespacedName, status *gatewayv1.RouteStatus) error { + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + original := &gatewayv1.Route{} + if err := r.Get(ctx, nn, original); err != nil { + return err + } + original.Status = *status + if err := r.Client.Status().Update(ctx, original); err != nil { + return err + } + return nil + }); err != nil { + return err + } + return nil +} + func getRouteId(route *gatewayv1.Route) string { return route.Namespace + "-" + route.Name } From 9bd6d071eff0101a09b8adb7e53f47bf43d49d68 Mon Sep 17 00:00:00 2001 From: october <764213885@qq.com> Date: Thu, 15 Sep 2022 23:03:46 +0800 Subject: [PATCH 7/7] feat(gateway): remove old code --- controllers/gateway/controllers/route_controller.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/controllers/gateway/controllers/route_controller.go b/controllers/gateway/controllers/route_controller.go index cdd4cea8ae..16e1e95004 100644 --- a/controllers/gateway/controllers/route_controller.go +++ b/controllers/gateway/controllers/route_controller.go @@ -57,12 +57,6 @@ type RouteReconciler struct { func (r *RouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { _ = log.FromContext(ctx) - // 获取apiSix集群操作对象 - apiSixCluster, result, err := r.getApiSixCluster(ctx, &gatewayv1.Route{}) - if err != nil { - return result, err - } - // get the route var route gatewayv1.Route if err := r.Get(ctx, req.NamespacedName, &route); err != nil {