diff --git a/.github/workflows/cicd-push.yml b/.github/workflows/cicd-push.yml index ca10014fc6f..6b2f5be7ecf 100644 --- a/.github/workflows/cicd-push.yml +++ b/.github/workflows/cicd-push.yml @@ -83,7 +83,7 @@ jobs: for filePath in $(echo "$FILE_PATH"); do filePath="${filePath%/*}" if [[ -d $filePath && "$filePath" != *"i18n/zh-cn"* ]]; then - echo $(pcregrep -r -n -I '[^\x00-\x7f]' $filePath >> pcregrep.out) + echo $(find "$filePath" -type f ! -name "*i18n*" -exec pcregrep -n -I '[^\x00-\x7f]' {} + >> pcregrep.out) fi done diff --git a/PROJECT b/PROJECT index 3d1ff5fd73d..11f4a2c7e1e 100644 --- a/PROJECT +++ b/PROJECT @@ -283,6 +283,16 @@ resources: kind: NodeCountScaler path: github.com/apecloud/kubeblocks/apis/experimental/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: kubeblocks.io + domain: kubeblocks.io + group: trace + kind: ReconciliationTrace + path: github.com/apecloud/kubeblocks/apis/trace/v1 + version: v1 - api: crdVersion: v1 namespaced: true diff --git a/apis/trace/v1/groupversion_info.go b/apis/trace/v1/groupversion_info.go new file mode 100644 index 00000000000..81971c67398 --- /dev/null +++ b/apis/trace/v1/groupversion_info.go @@ -0,0 +1,41 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +// Package v1 contains API Schema definitions for the trace v1 API group +// +kubebuilder:object:generate=true +// +groupName=trace.kubeblocks.io +package v1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "trace.kubeblocks.io", Version: "v1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) + +const Kind = "ReconciliationTrace" diff --git a/apis/trace/v1/reconciliationtrace_types.go b/apis/trace/v1/reconciliationtrace_types.go new file mode 100644 index 00000000000..287eefd7339 --- /dev/null +++ b/apis/trace/v1/reconciliationtrace_types.go @@ -0,0 +1,379 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package v1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ReconciliationTraceSpec defines the desired state of ReconciliationTrace +type ReconciliationTraceSpec struct { + // TargetObject specifies the target Cluster object. + // Default is the Cluster object with same namespace and name as this ReconciliationTrace object. + // + // +optional + TargetObject *ObjectReference `json:"targetObject,omitempty"` + + // DryRun tells the Controller to simulate the reconciliation process with a new desired spec of the TargetObject. + // And a reconciliation plan will be generated and described in the ReconciliationTraceStatus. + // The plan generation process will not impact the state of the TargetObject. + // + // +optional + DryRun *DryRun `json:"dryRun,omitempty"` + + // StateEvaluationExpression specifies the state evaluation expression used during reconciliation progress observation. + // The whole reconciliation process from the creation of the TargetObject to the deletion of it + // is separated into several reconciliation cycles. + // The StateEvaluationExpression is applied to the TargetObject, + // and an evaluation result of true indicates the end of a reconciliation cycle. + // StateEvaluationExpression overrides the builtin default value. + // + // +optional + StateEvaluationExpression *StateEvaluationExpression `json:"stateEvaluationExpression,omitempty"` + + // Locale specifies the locale to use when localizing the reconciliation trace. + // + // +optional + Locale *string `json:"locale,omitempty"` +} + +// ReconciliationTraceStatus defines the observed state of ReconciliationTrace +type ReconciliationTraceStatus struct { + // DryRunResult specifies the dry-run result. + // + // +optional + DryRunResult *DryRunResult `json:"dryRunResult,omitempty"` + + // InitialObjectTree specifies the initial object tree when the latest reconciliation cycle started. + // + InitialObjectTree *ObjectTreeNode `json:"initialObjectTree"` + + // CurrentState is the current state of the latest reconciliation cycle, + // that is the reconciliation process from the end of last reconciliation cycle until now. + // + CurrentState ReconciliationCycleState `json:"currentState"` + + // DesiredState is the desired state of the latest reconciliation cycle. + // + // +optional + DesiredState *ReconciliationCycleState `json:"desiredState,omitempty"` +} + +// ObjectReference defines a reference to an object. +type ObjectReference struct { + // Namespace of the referent. + // Default is same as the ReconciliationTrace object. + // + // +optional + Namespace string `json:"namespace,omitempty"` + + // Name of the referent. + // Default is same as the ReconciliationTrace object. + // + // +optional + Name string `json:"name,omitempty"` +} + +type DryRun struct { + // DesiredSpec specifies the desired spec of the TargetObject. + // The desired spec will be merged into the current spec by a strategic merge patch way to build the final spec, + // and the reconciliation plan will be calculated by comparing the current spec to the final spec. + // DesiredSpec should be a valid YAML string. + // + DesiredSpec string `json:"desiredSpec"` +} + +// StateEvaluationExpression defines an object state evaluation expression. +// Currently supported types: +// CEL - Common Expression Language (https://cel.dev/). +type StateEvaluationExpression struct { + // CELExpression specifies to use CEL to evaluation the object state. + // The root object used in the expression is the primary object. + // + // +optional + CELExpression *CELExpression `json:"celExpression,omitempty"` +} + +// CELExpression defines a CEL expression. +type CELExpression struct { + // Expression specifies the CEL expression. + // + Expression string `json:"expression"` +} + +// DryRunResult defines a dry-run result. +type DryRunResult struct { + // Phase specifies the current phase of the plan generation process. + // Succeed - the plan is calculated successfully. + // Failed - the plan can't be generated for some reason described in Reason. + // + // +kubebuilder:validation:Enum={Succeed,Failed} + Phase DryRunPhase `json:"phase,omitempty"` + + // Reason specifies the reason when the Phase is Failed. + // + // +optional + Reason string `json:"reason,omitempty"` + + // Message specifies a description of the failure reason. + // + // +optional + Message string `json:"message,omitempty"` + + // DesiredSpecRevision specifies the revision of the DesiredSpec. + // + DesiredSpecRevision string `json:"desiredSpecRevision"` + + // ObservedTargetGeneration specifies the observed generation of the TargetObject. + // + ObservedTargetGeneration int64 `json:"observedTargetGeneration"` + + // SpecDiff describes the diff between the current spec and the final spec. + // The whole spec struct will be compared and an example SpecDiff looks like: + // { + // Affinity: { + // PodAntiAffinity: "Preferred", + // Tenancy: "SharedNode", + // }, + // ComponentSpecs: { + // { + // ComponentDef: "postgresql", + // Name: "postgresql", + // - Replicas: 2, + // + Replicas: 3, + // Resources: + // { + // Limits: + // { + // - CPU: 500m, + // + CPU: 800m, + // - Memory: 512Mi, + // + Memory: 768Mi, + // }, + // Requests: + // { + // - CPU: 500m, + // + CPU: 800m, + // - Memory: 512Mi, + // + Memory: 768Mi, + // }, + // }, + // }, + // }, + // } + // + SpecDiff string `json:"specDiff"` + + // Plan describes the detail reconciliation process if the DesiredSpec is applied. + // + Plan ReconciliationCycleState `json:"plan"` +} + +type DryRunPhase string + +const ( + DryRunSucceedPhase DryRunPhase = "Succeed" + DryRunFailedPhase DryRunPhase = "Failed" +) + +// ObjectTreeNode defines an object tree of the KubeBlocks Cluster. +type ObjectTreeNode struct { + // Primary specifies reference of the primary object. + // + Primary corev1.ObjectReference `json:"primary"` + + // Secondaries describes all the secondary objects of this object, if any. + // + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + // +optional + Secondaries []*ObjectTreeNode `json:"secondaries,omitempty"` +} + +// ObjectSummary defines the total and change of an object. +type ObjectSummary struct { + // ObjectType of the object. + // + ObjectType ObjectType `json:"objectType"` + + // Total number of the object of type defined by ObjectType. + // + Total int32 `json:"total"` + + // ChangeSummary summarizes the change by comparing the final state to the current state of this type. + // Nil means no change. + // + // +optional + ChangeSummary *ObjectChangeSummary `json:"changeSummary,omitempty"` +} + +// ObjectType defines an object type. +type ObjectType struct { + // APIVersion of the type. + // + APIVersion string `json:"apiVersion"` + + // Kind of the type. + // + Kind string `json:"kind"` +} + +// ObjectChangeSummary defines changes of an object. +type ObjectChangeSummary struct { + // Added specifies the number of object will be added. + // + // +optional + Added *int32 `json:"added,omitempty"` + + // Updated specifies the number of object will be updated. + // + // +optional + Updated *int32 `json:"updated,omitempty"` + + // Deleted specifies the number of object will be deleted. + // + // +optional + Deleted *int32 `json:"deleted,omitempty"` +} + +// ObjectChange defines a detailed change of an object. +type ObjectChange struct { + // ObjectReference specifies the Object this change described. + // + ObjectReference corev1.ObjectReference `json:"objectReference"` + + // ChangeType specifies the change type. + // Event - specifies that this is a Kubernetes Event. + // Creation - specifies that this is an object creation. + // Update - specifies that this is an object update. + // Deletion - specifies that this is an object deletion. + // + // +kubebuilder:validation:Enum={Event, Creation, Update, Deletion} + ChangeType ObjectChangeType `json:"changeType"` + + // EventAttributes specifies the attributes of the event when ChangeType is Event. + // + // +optional + EventAttributes *EventAttributes `json:"eventAttributes,omitempty"` + + // Revision specifies the revision of the object after this change. + // Revision can be compared globally between all ObjectChanges of all Objects, to build a total order object change sequence. + // + Revision int64 `json:"revision"` + + // Timestamp is a timestamp representing the ReconciliationTrace Controller time when this change occurred. + // It is not guaranteed to be set in happens-before order across separate changes. + // It is represented in RFC3339 form and is in UTC. + // + // +optional + Timestamp *metav1.Time `json:"timestamp,omitempty"` + + // Description describes the change in a user-friendly way. + // + Description string `json:"description"` + + // LocalDescription is the localized version of Description by using the Locale specified in `spec.locale`. + // Empty if the `spec.locale` is not specified. + // + LocalDescription *string `json:"localDescription,omitempty"` +} + +type ObjectChangeType string + +const ( + ObjectCreationType ObjectChangeType = "Creation" + ObjectUpdateType ObjectChangeType = "Update" + ObjectDeletionType ObjectChangeType = "Deletion" + EventType ObjectChangeType = "Event" +) + +// EventAttributes defines attributes of the Event. +type EventAttributes struct { + // Name of the Event. + // + Name string `json:"name"` + + // Type of the Event. + // + Type string `json:"type"` + + // Reason of the Event. + // + Reason string `json:"reason"` +} + +// ReconciliationCycleState defines the state of reconciliation cycle. +type ReconciliationCycleState struct { + // Summary summarizes the ObjectTree and Changes. + // + Summary ObjectTreeDiffSummary `json:"summary"` + + // ObjectTree specifies the current object tree of the reconciliation cycle. + // Ideally, ObjectTree should be same as applying Changes to InitialObjectTree. + // + ObjectTree *ObjectTreeNode `json:"objectTree"` + + // Changes describes the detail reconciliation process. + // + Changes []ObjectChange `json:"changes"` +} + +// ObjectTreeDiffSummary defines a summary of the diff of two object tree. +type ObjectTreeDiffSummary struct { + // ObjectSummaries summarizes each object type. + // + ObjectSummaries []ObjectSummary `json:"objectSummaries"` +} + +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:categories={kubeblocks,all},shortName=trace +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="TARGET_NS",type="string",JSONPath=".spec.targetObject.namespace",description="Target Object Namespace" +// +kubebuilder:printcolumn:name="TARGET_NAME",type="string",JSONPath=".spec.targetObject.name",description="Target Object Name" +// +kubebuilder:printcolumn:name="API_VERSION",type="string",JSONPath=".status.currentState.changes[-1].objectReference.apiVersion",description="Latest Changed Object API Version" +// +kubebuilder:printcolumn:name="KIND",type="string",JSONPath=".status.currentState.changes[-1].objectReference.kind",description="Latest Changed Object Kind" +// +kubebuilder:printcolumn:name="NAMESPACE",type="string",JSONPath=".status.currentState.changes[-1].objectReference.namespace",description="Latest Changed Object Namespace" +// +kubebuilder:printcolumn:name="NAME",type="string",JSONPath=".status.currentState.changes[-1].objectReference.name",description="Latest Changed Object Name" +// +kubebuilder:printcolumn:name="CHANGE",type="string",JSONPath=".status.currentState.changes[-1].description",description="Latest Change Description" + +// ReconciliationTrace is the Schema for the reconciliationtraces API +type ReconciliationTrace struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ReconciliationTraceSpec `json:"spec,omitempty"` + Status ReconciliationTraceStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// ReconciliationTraceList contains a list of ReconciliationTrace +type ReconciliationTraceList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ReconciliationTrace `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ReconciliationTrace{}, &ReconciliationTraceList{}) +} diff --git a/apis/trace/v1/zz_generated.deepcopy.go b/apis/trace/v1/zz_generated.deepcopy.go new file mode 100644 index 00000000000..79b8fb7dcf7 --- /dev/null +++ b/apis/trace/v1/zz_generated.deepcopy.go @@ -0,0 +1,422 @@ +//go:build !ignore_autogenerated + +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CELExpression) DeepCopyInto(out *CELExpression) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CELExpression. +func (in *CELExpression) DeepCopy() *CELExpression { + if in == nil { + return nil + } + out := new(CELExpression) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DryRun) DeepCopyInto(out *DryRun) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DryRun. +func (in *DryRun) DeepCopy() *DryRun { + if in == nil { + return nil + } + out := new(DryRun) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DryRunResult) DeepCopyInto(out *DryRunResult) { + *out = *in + in.Plan.DeepCopyInto(&out.Plan) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DryRunResult. +func (in *DryRunResult) DeepCopy() *DryRunResult { + if in == nil { + return nil + } + out := new(DryRunResult) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EventAttributes) DeepCopyInto(out *EventAttributes) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EventAttributes. +func (in *EventAttributes) DeepCopy() *EventAttributes { + if in == nil { + return nil + } + out := new(EventAttributes) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectChange) DeepCopyInto(out *ObjectChange) { + *out = *in + out.ObjectReference = in.ObjectReference + if in.EventAttributes != nil { + in, out := &in.EventAttributes, &out.EventAttributes + *out = new(EventAttributes) + **out = **in + } + if in.Timestamp != nil { + in, out := &in.Timestamp, &out.Timestamp + *out = (*in).DeepCopy() + } + if in.LocalDescription != nil { + in, out := &in.LocalDescription, &out.LocalDescription + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectChange. +func (in *ObjectChange) DeepCopy() *ObjectChange { + if in == nil { + return nil + } + out := new(ObjectChange) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectChangeSummary) DeepCopyInto(out *ObjectChangeSummary) { + *out = *in + if in.Added != nil { + in, out := &in.Added, &out.Added + *out = new(int32) + **out = **in + } + if in.Updated != nil { + in, out := &in.Updated, &out.Updated + *out = new(int32) + **out = **in + } + if in.Deleted != nil { + in, out := &in.Deleted, &out.Deleted + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectChangeSummary. +func (in *ObjectChangeSummary) DeepCopy() *ObjectChangeSummary { + if in == nil { + return nil + } + out := new(ObjectChangeSummary) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectReference) DeepCopyInto(out *ObjectReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectReference. +func (in *ObjectReference) DeepCopy() *ObjectReference { + if in == nil { + return nil + } + out := new(ObjectReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectSummary) DeepCopyInto(out *ObjectSummary) { + *out = *in + out.ObjectType = in.ObjectType + if in.ChangeSummary != nil { + in, out := &in.ChangeSummary, &out.ChangeSummary + *out = new(ObjectChangeSummary) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectSummary. +func (in *ObjectSummary) DeepCopy() *ObjectSummary { + if in == nil { + return nil + } + out := new(ObjectSummary) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectTreeDiffSummary) DeepCopyInto(out *ObjectTreeDiffSummary) { + *out = *in + if in.ObjectSummaries != nil { + in, out := &in.ObjectSummaries, &out.ObjectSummaries + *out = make([]ObjectSummary, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectTreeDiffSummary. +func (in *ObjectTreeDiffSummary) DeepCopy() *ObjectTreeDiffSummary { + if in == nil { + return nil + } + out := new(ObjectTreeDiffSummary) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectTreeNode) DeepCopyInto(out *ObjectTreeNode) { + *out = *in + out.Primary = in.Primary + if in.Secondaries != nil { + in, out := &in.Secondaries, &out.Secondaries + *out = make([]*ObjectTreeNode, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(ObjectTreeNode) + (*in).DeepCopyInto(*out) + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectTreeNode. +func (in *ObjectTreeNode) DeepCopy() *ObjectTreeNode { + if in == nil { + return nil + } + out := new(ObjectTreeNode) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectType) DeepCopyInto(out *ObjectType) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectType. +func (in *ObjectType) DeepCopy() *ObjectType { + if in == nil { + return nil + } + out := new(ObjectType) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReconciliationCycleState) DeepCopyInto(out *ReconciliationCycleState) { + *out = *in + in.Summary.DeepCopyInto(&out.Summary) + if in.ObjectTree != nil { + in, out := &in.ObjectTree, &out.ObjectTree + *out = new(ObjectTreeNode) + (*in).DeepCopyInto(*out) + } + if in.Changes != nil { + in, out := &in.Changes, &out.Changes + *out = make([]ObjectChange, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReconciliationCycleState. +func (in *ReconciliationCycleState) DeepCopy() *ReconciliationCycleState { + if in == nil { + return nil + } + out := new(ReconciliationCycleState) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReconciliationTrace) DeepCopyInto(out *ReconciliationTrace) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReconciliationTrace. +func (in *ReconciliationTrace) DeepCopy() *ReconciliationTrace { + if in == nil { + return nil + } + out := new(ReconciliationTrace) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ReconciliationTrace) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReconciliationTraceList) DeepCopyInto(out *ReconciliationTraceList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ReconciliationTrace, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReconciliationTraceList. +func (in *ReconciliationTraceList) DeepCopy() *ReconciliationTraceList { + if in == nil { + return nil + } + out := new(ReconciliationTraceList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ReconciliationTraceList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReconciliationTraceSpec) DeepCopyInto(out *ReconciliationTraceSpec) { + *out = *in + if in.TargetObject != nil { + in, out := &in.TargetObject, &out.TargetObject + *out = new(ObjectReference) + **out = **in + } + if in.DryRun != nil { + in, out := &in.DryRun, &out.DryRun + *out = new(DryRun) + **out = **in + } + if in.StateEvaluationExpression != nil { + in, out := &in.StateEvaluationExpression, &out.StateEvaluationExpression + *out = new(StateEvaluationExpression) + (*in).DeepCopyInto(*out) + } + if in.Locale != nil { + in, out := &in.Locale, &out.Locale + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReconciliationTraceSpec. +func (in *ReconciliationTraceSpec) DeepCopy() *ReconciliationTraceSpec { + if in == nil { + return nil + } + out := new(ReconciliationTraceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReconciliationTraceStatus) DeepCopyInto(out *ReconciliationTraceStatus) { + *out = *in + if in.DryRunResult != nil { + in, out := &in.DryRunResult, &out.DryRunResult + *out = new(DryRunResult) + (*in).DeepCopyInto(*out) + } + if in.InitialObjectTree != nil { + in, out := &in.InitialObjectTree, &out.InitialObjectTree + *out = new(ObjectTreeNode) + (*in).DeepCopyInto(*out) + } + in.CurrentState.DeepCopyInto(&out.CurrentState) + if in.DesiredState != nil { + in, out := &in.DesiredState, &out.DesiredState + *out = new(ReconciliationCycleState) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReconciliationTraceStatus. +func (in *ReconciliationTraceStatus) DeepCopy() *ReconciliationTraceStatus { + if in == nil { + return nil + } + out := new(ReconciliationTraceStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StateEvaluationExpression) DeepCopyInto(out *StateEvaluationExpression) { + *out = *in + if in.CELExpression != nil { + in, out := &in.CELExpression, &out.CELExpression + *out = new(CELExpression) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StateEvaluationExpression. +func (in *StateEvaluationExpression) DeepCopy() *StateEvaluationExpression { + if in == nil { + return nil + } + out := new(StateEvaluationExpression) + in.DeepCopyInto(out) + return out +} diff --git a/apis/workloads/v1/groupversion_info.go b/apis/workloads/v1/groupversion_info.go index 61b08ee95f1..f5d8d535d51 100644 --- a/apis/workloads/v1/groupversion_info.go +++ b/apis/workloads/v1/groupversion_info.go @@ -38,4 +38,4 @@ var ( AddToScheme = SchemeBuilder.AddToScheme ) -const Kind = "InstanceSet" +const InstanceSetKind = "InstanceSet" diff --git a/cmd/dataprotection/main.go b/cmd/dataprotection/main.go index 1eddcca05e1..a36ff371241 100644 --- a/cmd/dataprotection/main.go +++ b/cmd/dataprotection/main.go @@ -207,7 +207,7 @@ func main() { if len(managedNamespaces) > 0 { setupLog.Info(fmt.Sprintf("managed namespaces: %s", managedNamespaces)) } - mgr, err := ctrl.NewManager(intctrlutil.GeKubeRestConfig(userAgent), ctrl.Options{ + mgr, err := ctrl.NewManager(intctrlutil.GetKubeRestConfig(userAgent), ctrl.Options{ Scheme: scheme, Metrics: server.Options{ BindAddress: metricsAddr, diff --git a/cmd/manager/main.go b/cmd/manager/main.go index f87af4d5045..0607c863e2e 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -55,6 +55,7 @@ import ( experimentalv1alpha1 "github.com/apecloud/kubeblocks/apis/experimental/v1alpha1" extensionsv1alpha1 "github.com/apecloud/kubeblocks/apis/extensions/v1alpha1" opsv1alpha1 "github.com/apecloud/kubeblocks/apis/operations/v1alpha1" + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" workloadsv1 "github.com/apecloud/kubeblocks/apis/workloads/v1" workloadsv1alpha1 "github.com/apecloud/kubeblocks/apis/workloads/v1alpha1" appscontrollers "github.com/apecloud/kubeblocks/controllers/apps" @@ -63,6 +64,7 @@ import ( extensionscontrollers "github.com/apecloud/kubeblocks/controllers/extensions" k8scorecontrollers "github.com/apecloud/kubeblocks/controllers/k8score" opscontrollers "github.com/apecloud/kubeblocks/controllers/operations" + tracecontrollers "github.com/apecloud/kubeblocks/controllers/trace" workloadscontrollers "github.com/apecloud/kubeblocks/controllers/workloads" "github.com/apecloud/kubeblocks/pkg/constant" "github.com/apecloud/kubeblocks/pkg/controller/instanceset" @@ -89,6 +91,7 @@ const ( operationsFlagKey flagName = "operations" extensionsFlagKey flagName = "extensions" experimentalFlagKey flagName = "experimental" + traceFlagKey flagName = "trace" multiClusterKubeConfigFlagKey flagName = "multi-cluster-kubeconfig" multiClusterContextsFlagKey flagName = "multi-cluster-contexts" @@ -115,6 +118,7 @@ func init() { utilruntime.Must(workloadsv1alpha1.AddToScheme(scheme)) utilruntime.Must(workloadsv1.AddToScheme(scheme)) utilruntime.Must(experimentalv1alpha1.AddToScheme(scheme)) + utilruntime.Must(tracev1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme @@ -144,6 +148,7 @@ func init() { viper.SetDefault(constant.CfgKBReconcileWorkers, 8) viper.SetDefault(constant.FeatureGateIgnoreConfigTemplateDefaultMode, false) viper.SetDefault(constant.FeatureGateInPlacePodVerticalScaling, false) + viper.SetDefault(constant.I18nResourcesName, "kubeblocks-i18n-resources") } type flagName string @@ -177,6 +182,8 @@ func setupFlags() { "Enable the extensions controller manager.") flag.Bool(experimentalFlagKey.String(), false, "Enable the experimental controller manager.") + flag.Bool(traceFlagKey.String(), true, + "Enable the trace controller manager.") flag.String(multiClusterKubeConfigFlagKey.String(), "", "Paths to the kubeconfig for multi-cluster accessing.") flag.String(multiClusterContextsFlagKey.String(), "", "Kube contexts the manager will talk to.") @@ -312,7 +319,7 @@ func main() { userAgent = viper.GetString(userAgentFlagKey.viperName()) setupLog.Info("golang runtime metrics.", "featureGate", intctrlutil.EnabledRuntimeMetrics()) - mgr, err := ctrl.NewManager(intctrlutil.GeKubeRestConfig(userAgent), ctrl.Options{ + mgr, err := ctrl.NewManager(intctrlutil.GetKubeRestConfig(userAgent), ctrl.Options{ Scheme: scheme, Metrics: server.Options{ BindAddress: metricsAddr, @@ -537,6 +544,22 @@ func main() { } } + if viper.GetBool(traceFlagKey.viperName()) { + traceReconciler := &tracecontrollers.ReconciliationTraceReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("reconciliation-trace-controller"), + } + if err := traceReconciler.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "ReconciliationTrace") + os.Exit(1) + } + if err := mgr.Add(traceReconciler.InformerManager); err != nil { + setupLog.Error(err, "unable to add trace informer manager", "controller", "InformerManager") + os.Exit(1) + } + } + if os.Getenv("ENABLE_WEBHOOKS") == "true" { if err = (&appsv1.ClusterDefinition{}).SetupWebhookWithManager(mgr); err != nil { setupLog.Error(err, "unable to create webhook", "webhook", "ClusterDefinition") diff --git a/config/crd/bases/trace.kubeblocks.io_reconciliationtraces.yaml b/config/crd/bases/trace.kubeblocks.io_reconciliationtraces.yaml new file mode 100644 index 00000000000..fc42f210251 --- /dev/null +++ b/config/crd/bases/trace.kubeblocks.io_reconciliationtraces.yaml @@ -0,0 +1,946 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + labels: + app.kubernetes.io/name: kubeblocks + name: reconciliationtraces.trace.kubeblocks.io +spec: + group: trace.kubeblocks.io + names: + categories: + - kubeblocks + - all + kind: ReconciliationTrace + listKind: ReconciliationTraceList + plural: reconciliationtraces + shortNames: + - trace + singular: reconciliationtrace + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + - description: Target Object Namespace + jsonPath: .spec.targetObject.namespace + name: TARGET_NS + type: string + - description: Target Object Name + jsonPath: .spec.targetObject.name + name: TARGET_NAME + type: string + - description: Latest Changed Object API Version + jsonPath: .status.currentState.changes[-1].objectReference.apiVersion + name: API_VERSION + type: string + - description: Latest Changed Object Kind + jsonPath: .status.currentState.changes[-1].objectReference.kind + name: KIND + type: string + - description: Latest Changed Object Namespace + jsonPath: .status.currentState.changes[-1].objectReference.namespace + name: NAMESPACE + type: string + - description: Latest Changed Object Name + jsonPath: .status.currentState.changes[-1].objectReference.name + name: NAME + type: string + - description: Latest Change Description + jsonPath: .status.currentState.changes[-1].description + name: CHANGE + type: string + name: v1 + schema: + openAPIV3Schema: + description: ReconciliationTrace is the Schema for the reconciliationtraces + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ReconciliationTraceSpec defines the desired state of ReconciliationTrace + properties: + dryRun: + description: |- + DryRun tells the Controller to simulate the reconciliation process with a new desired spec of the TargetObject. + And a reconciliation plan will be generated and described in the ReconciliationTraceStatus. + The plan generation process will not impact the state of the TargetObject. + properties: + desiredSpec: + description: |- + DesiredSpec specifies the desired spec of the TargetObject. + The desired spec will be merged into the current spec by a strategic merge patch way to build the final spec, + and the reconciliation plan will be calculated by comparing the current spec to the final spec. + DesiredSpec should be a valid YAML string. + type: string + required: + - desiredSpec + type: object + locale: + description: Locale specifies the locale to use when localizing the + reconciliation trace. + type: string + stateEvaluationExpression: + description: |- + StateEvaluationExpression specifies the state evaluation expression used during reconciliation progress observation. + The whole reconciliation process from the creation of the TargetObject to the deletion of it + is separated into several reconciliation cycles. + The StateEvaluationExpression is applied to the TargetObject, + and an evaluation result of true indicates the end of a reconciliation cycle. + StateEvaluationExpression overrides the builtin default value. + properties: + celExpression: + description: |- + CELExpression specifies to use CEL to evaluation the object state. + The root object used in the expression is the primary object. + properties: + expression: + description: Expression specifies the CEL expression. + type: string + required: + - expression + type: object + type: object + targetObject: + description: |- + TargetObject specifies the target Cluster object. + Default is the Cluster object with same namespace and name as this ReconciliationTrace object. + properties: + name: + description: |- + Name of the referent. + Default is same as the ReconciliationTrace object. + type: string + namespace: + description: |- + Namespace of the referent. + Default is same as the ReconciliationTrace object. + type: string + type: object + type: object + status: + description: ReconciliationTraceStatus defines the observed state of ReconciliationTrace + properties: + currentState: + description: |- + CurrentState is the current state of the latest reconciliation cycle, + that is the reconciliation process from the end of last reconciliation cycle until now. + properties: + changes: + description: Changes describes the detail reconciliation process. + items: + description: ObjectChange defines a detailed change of an object. + properties: + changeType: + description: |- + ChangeType specifies the change type. + Event - specifies that this is a Kubernetes Event. + Creation - specifies that this is an object creation. + Update - specifies that this is an object update. + Deletion - specifies that this is an object deletion. + enum: + - Event + - Creation + - Update + - Deletion + type: string + description: + description: Description describes the change in a user-friendly + way. + type: string + eventAttributes: + description: EventAttributes specifies the attributes of + the event when ChangeType is Event. + properties: + name: + description: Name of the Event. + type: string + reason: + description: Reason of the Event. + type: string + type: + description: Type of the Event. + type: string + required: + - name + - reason + - type + type: object + localDescription: + description: |- + LocalDescription is the localized version of Description by using the Locale specified in `spec.locale`. + Empty if the `spec.locale` is not specified. + type: string + objectReference: + description: ObjectReference specifies the Object this change + described. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + revision: + description: |- + Revision specifies the revision of the object after this change. + Revision can be compared globally between all ObjectChanges of all Objects, to build a total order object change sequence. + format: int64 + type: integer + timestamp: + description: |- + Timestamp is a timestamp representing the ReconciliationTrace Controller time when this change occurred. + It is not guaranteed to be set in happens-before order across separate changes. + It is represented in RFC3339 form and is in UTC. + format: date-time + type: string + required: + - changeType + - description + - objectReference + - revision + type: object + type: array + objectTree: + description: |- + ObjectTree specifies the current object tree of the reconciliation cycle. + Ideally, ObjectTree should be same as applying Changes to InitialObjectTree. + properties: + primary: + description: Primary specifies reference of the primary object. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + secondaries: + description: Secondaries describes all the secondary objects + of this object, if any. + x-kubernetes-preserve-unknown-fields: true + required: + - primary + type: object + summary: + description: Summary summarizes the ObjectTree and Changes. + properties: + objectSummaries: + description: ObjectSummaries summarizes each object type. + items: + description: ObjectSummary defines the total and change + of an object. + properties: + changeSummary: + description: |- + ChangeSummary summarizes the change by comparing the final state to the current state of this type. + Nil means no change. + properties: + added: + description: Added specifies the number of object + will be added. + format: int32 + type: integer + deleted: + description: Deleted specifies the number of object + will be deleted. + format: int32 + type: integer + updated: + description: Updated specifies the number of object + will be updated. + format: int32 + type: integer + type: object + objectType: + description: ObjectType of the object. + properties: + apiVersion: + description: APIVersion of the type. + type: string + kind: + description: Kind of the type. + type: string + required: + - apiVersion + - kind + type: object + total: + description: Total number of the object of type defined + by ObjectType. + format: int32 + type: integer + required: + - objectType + - total + type: object + type: array + required: + - objectSummaries + type: object + required: + - changes + - objectTree + - summary + type: object + desiredState: + description: DesiredState is the desired state of the latest reconciliation + cycle. + properties: + changes: + description: Changes describes the detail reconciliation process. + items: + description: ObjectChange defines a detailed change of an object. + properties: + changeType: + description: |- + ChangeType specifies the change type. + Event - specifies that this is a Kubernetes Event. + Creation - specifies that this is an object creation. + Update - specifies that this is an object update. + Deletion - specifies that this is an object deletion. + enum: + - Event + - Creation + - Update + - Deletion + type: string + description: + description: Description describes the change in a user-friendly + way. + type: string + eventAttributes: + description: EventAttributes specifies the attributes of + the event when ChangeType is Event. + properties: + name: + description: Name of the Event. + type: string + reason: + description: Reason of the Event. + type: string + type: + description: Type of the Event. + type: string + required: + - name + - reason + - type + type: object + localDescription: + description: |- + LocalDescription is the localized version of Description by using the Locale specified in `spec.locale`. + Empty if the `spec.locale` is not specified. + type: string + objectReference: + description: ObjectReference specifies the Object this change + described. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + revision: + description: |- + Revision specifies the revision of the object after this change. + Revision can be compared globally between all ObjectChanges of all Objects, to build a total order object change sequence. + format: int64 + type: integer + timestamp: + description: |- + Timestamp is a timestamp representing the ReconciliationTrace Controller time when this change occurred. + It is not guaranteed to be set in happens-before order across separate changes. + It is represented in RFC3339 form and is in UTC. + format: date-time + type: string + required: + - changeType + - description + - objectReference + - revision + type: object + type: array + objectTree: + description: |- + ObjectTree specifies the current object tree of the reconciliation cycle. + Ideally, ObjectTree should be same as applying Changes to InitialObjectTree. + properties: + primary: + description: Primary specifies reference of the primary object. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + secondaries: + description: Secondaries describes all the secondary objects + of this object, if any. + x-kubernetes-preserve-unknown-fields: true + required: + - primary + type: object + summary: + description: Summary summarizes the ObjectTree and Changes. + properties: + objectSummaries: + description: ObjectSummaries summarizes each object type. + items: + description: ObjectSummary defines the total and change + of an object. + properties: + changeSummary: + description: |- + ChangeSummary summarizes the change by comparing the final state to the current state of this type. + Nil means no change. + properties: + added: + description: Added specifies the number of object + will be added. + format: int32 + type: integer + deleted: + description: Deleted specifies the number of object + will be deleted. + format: int32 + type: integer + updated: + description: Updated specifies the number of object + will be updated. + format: int32 + type: integer + type: object + objectType: + description: ObjectType of the object. + properties: + apiVersion: + description: APIVersion of the type. + type: string + kind: + description: Kind of the type. + type: string + required: + - apiVersion + - kind + type: object + total: + description: Total number of the object of type defined + by ObjectType. + format: int32 + type: integer + required: + - objectType + - total + type: object + type: array + required: + - objectSummaries + type: object + required: + - changes + - objectTree + - summary + type: object + dryRunResult: + description: DryRunResult specifies the dry-run result. + properties: + desiredSpecRevision: + description: DesiredSpecRevision specifies the revision of the + DesiredSpec. + type: string + message: + description: Message specifies a description of the failure reason. + type: string + observedTargetGeneration: + description: ObservedTargetGeneration specifies the observed generation + of the TargetObject. + format: int64 + type: integer + phase: + description: |- + Phase specifies the current phase of the plan generation process. + Succeed - the plan is calculated successfully. + Failed - the plan can't be generated for some reason described in Reason. + enum: + - Succeed + - Failed + type: string + plan: + description: Plan describes the detail reconciliation process + if the DesiredSpec is applied. + properties: + changes: + description: Changes describes the detail reconciliation process. + items: + description: ObjectChange defines a detailed change of an + object. + properties: + changeType: + description: |- + ChangeType specifies the change type. + Event - specifies that this is a Kubernetes Event. + Creation - specifies that this is an object creation. + Update - specifies that this is an object update. + Deletion - specifies that this is an object deletion. + enum: + - Event + - Creation + - Update + - Deletion + type: string + description: + description: Description describes the change in a user-friendly + way. + type: string + eventAttributes: + description: EventAttributes specifies the attributes + of the event when ChangeType is Event. + properties: + name: + description: Name of the Event. + type: string + reason: + description: Reason of the Event. + type: string + type: + description: Type of the Event. + type: string + required: + - name + - reason + - type + type: object + localDescription: + description: |- + LocalDescription is the localized version of Description by using the Locale specified in `spec.locale`. + Empty if the `spec.locale` is not specified. + type: string + objectReference: + description: ObjectReference specifies the Object this + change described. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + revision: + description: |- + Revision specifies the revision of the object after this change. + Revision can be compared globally between all ObjectChanges of all Objects, to build a total order object change sequence. + format: int64 + type: integer + timestamp: + description: |- + Timestamp is a timestamp representing the ReconciliationTrace Controller time when this change occurred. + It is not guaranteed to be set in happens-before order across separate changes. + It is represented in RFC3339 form and is in UTC. + format: date-time + type: string + required: + - changeType + - description + - objectReference + - revision + type: object + type: array + objectTree: + description: |- + ObjectTree specifies the current object tree of the reconciliation cycle. + Ideally, ObjectTree should be same as applying Changes to InitialObjectTree. + properties: + primary: + description: Primary specifies reference of the primary + object. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + secondaries: + description: Secondaries describes all the secondary objects + of this object, if any. + x-kubernetes-preserve-unknown-fields: true + required: + - primary + type: object + summary: + description: Summary summarizes the ObjectTree and Changes. + properties: + objectSummaries: + description: ObjectSummaries summarizes each object type. + items: + description: ObjectSummary defines the total and change + of an object. + properties: + changeSummary: + description: |- + ChangeSummary summarizes the change by comparing the final state to the current state of this type. + Nil means no change. + properties: + added: + description: Added specifies the number of object + will be added. + format: int32 + type: integer + deleted: + description: Deleted specifies the number of + object will be deleted. + format: int32 + type: integer + updated: + description: Updated specifies the number of + object will be updated. + format: int32 + type: integer + type: object + objectType: + description: ObjectType of the object. + properties: + apiVersion: + description: APIVersion of the type. + type: string + kind: + description: Kind of the type. + type: string + required: + - apiVersion + - kind + type: object + total: + description: Total number of the object of type + defined by ObjectType. + format: int32 + type: integer + required: + - objectType + - total + type: object + type: array + required: + - objectSummaries + type: object + required: + - changes + - objectTree + - summary + type: object + reason: + description: Reason specifies the reason when the Phase is Failed. + type: string + specDiff: + description: "SpecDiff describes the diff between the current + spec and the final spec.\nThe whole spec struct will be compared + and an example SpecDiff looks like:\n{\n \tAffinity: {\n \t\tPodAntiAffinity: + \"Preferred\",\n \t\tTenancy: \"SharedNode\",\n \t},\n \tComponentSpecs: + {\n \t\t{\n \t\t\tComponentDef: \"postgresql\",\n \t\t\tName: + \"postgresql\",\n-\t\t\tReplicas: 2,\n \t\t\tResources:\n \t\t\t{\n + \t\t\t\tLimits:\n \t\t\t\t{\n-\t\t\t\t\tCPU: 500m,\n-\t\t\t\t\tMemory: + 512Mi,\n \t\t\t\t},\n \t\t\t\tRequests:\n \t\t\t\t{\n-\t\t\t\t\tCPU: + 500m,\n-\t\t\t\t\tMemory: 512Mi,\n \t\t\t\t},\n \t\t\t},\n \t\t},\n + \t},\n}" + type: string + required: + - desiredSpecRevision + - observedTargetGeneration + - plan + - specDiff + type: object + initialObjectTree: + description: InitialObjectTree specifies the initial object tree when + the latest reconciliation cycle started. + properties: + primary: + description: Primary specifies reference of the primary object. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + secondaries: + description: Secondaries describes all the secondary objects of + this object, if any. + x-kubernetes-preserve-unknown-fields: true + required: + - primary + type: object + required: + - currentState + - initialObjectTree + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index bb22d74c59e..ea0f8d4c116 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -23,6 +23,7 @@ resources: - bases/experimental.kubeblocks.io_nodecountscalers.yaml - bases/operations.kubeblocks.io_opsrequests.yaml - bases/operations.kubeblocks.io_opsdefinitions.yaml +- bases/trace.kubeblocks.io_reconciliationtraces.yaml - bases/apps.kubeblocks.io_shardingdefinitions.yaml - bases/apps.kubeblocks.io_sidecardefinitions.yaml #+kubebuilder:scaffold:crdkustomizeresource @@ -53,6 +54,7 @@ patchesStrategicMerge: #- patches/webhook_in_opsdefinitions.yaml #- patches/webhook_in_componentversions.yaml #- patches/webhook_in_nodecountscalers.yaml +#- patches/webhook_in_reconciliationtraces.yaml #- patches/webhook_in_shardingdefinitions.yaml #- patches/webhook_in_sidecardefinitions.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch @@ -82,6 +84,7 @@ patchesStrategicMerge: #- patches/cainjection_in_opsdefinitions.yaml #- patches/cainjection_in_componentversions.yaml #- patches/cainjection_in_nodecountscalers.yaml +#- patches/cainjection_in_reconciliationtraces.yaml #- patches/cainjection_in_shardingdefinitions.yaml #- patches/cainjection_in_sidecardefinitions.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch diff --git a/config/crd/patches/cainjection_in_trace_reconciliationtraces.yaml b/config/crd/patches/cainjection_in_trace_reconciliationtraces.yaml new file mode 100644 index 00000000000..6d8192405dc --- /dev/null +++ b/config/crd/patches/cainjection_in_trace_reconciliationtraces.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: reconciliationviews.view.kubeblocks.io diff --git a/config/crd/patches/webhook_in_trace_reconciliationtraces.yaml b/config/crd/patches/webhook_in_trace_reconciliationtraces.yaml new file mode 100644 index 00000000000..85dad708747 --- /dev/null +++ b/config/crd/patches/webhook_in_trace_reconciliationtraces.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: reconciliationviews.view.kubeblocks.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 0c3fac2cabf..a6e1f3dde58 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -937,6 +937,32 @@ rules: - get - list - watch +- apiGroups: + - trace.kubeblocks.io + resources: + - reconciliationtraces + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - trace.kubeblocks.io + resources: + - reconciliationtraces/finalizers + verbs: + - update +- apiGroups: + - trace.kubeblocks.io + resources: + - reconciliationtraces/status + verbs: + - get + - patch + - update - apiGroups: - workloads.kubeblocks.io resources: diff --git a/config/rbac/trace_reconciliationtrace_editor_role.yaml b/config/rbac/trace_reconciliationtrace_editor_role.yaml new file mode 100644 index 00000000000..1fa16107de5 --- /dev/null +++ b/config/rbac/trace_reconciliationtrace_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit reconciliationviews. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: reconciliationview-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: kubeblocks + app.kubernetes.io/part-of: kubeblocks + app.kubernetes.io/managed-by: kustomize + name: reconciliationview-editor-role +rules: +- apiGroups: + - view.kubeblocks.io + resources: + - reconciliationviews + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - view.kubeblocks.io + resources: + - reconciliationviews/status + verbs: + - get diff --git a/config/rbac/trace_reconciliationtrace_viewer_role.yaml b/config/rbac/trace_reconciliationtrace_viewer_role.yaml new file mode 100644 index 00000000000..0d4be7079fb --- /dev/null +++ b/config/rbac/trace_reconciliationtrace_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view reconciliationviews. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: reconciliationview-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: kubeblocks + app.kubernetes.io/part-of: kubeblocks + app.kubernetes.io/managed-by: kustomize + name: reconciliationview-viewer-role +rules: +- apiGroups: + - view.kubeblocks.io + resources: + - reconciliationviews + verbs: + - get + - list + - watch +- apiGroups: + - view.kubeblocks.io + resources: + - reconciliationviews/status + verbs: + - get diff --git a/config/samples/trace_v1_reconciliationtrace.yaml b/config/samples/trace_v1_reconciliationtrace.yaml new file mode 100644 index 00000000000..1320a1b3fb8 --- /dev/null +++ b/config/samples/trace_v1_reconciliationtrace.yaml @@ -0,0 +1,12 @@ +apiVersion: trace.kubeblocks.io/v1 +kind: ReconciliationTrace +metadata: + labels: + app.kubernetes.io/name: reconciliationtrace + app.kubernetes.io/instance: reconciliationtrace-sample + app.kubernetes.io/part-of: kubeblocks + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: kubeblocks + name: reconciliationtrace-sample +spec: + # TODO(user): Add fields here diff --git a/controllers/apps/component_utils_test.go b/controllers/apps/component_utils_test.go index cc90b46ad54..62368327f11 100644 --- a/controllers/apps/component_utils_test.go +++ b/controllers/apps/component_utils_test.go @@ -69,7 +69,7 @@ var _ = Describe("Component Utils", func() { ) pod := testapps.MockInstanceSetPod(&testCtx, nil, clusterName, compName, podName, role, mode) ppod := testapps.NewPodFactory(testCtx.DefaultNamespace, "pod"). - SetOwnerReferences(workloads.GroupVersion.String(), workloads.Kind, nil). + SetOwnerReferences(workloads.GroupVersion.String(), workloads.InstanceSetKind, nil). AddAppInstanceLabel(clusterName). AddAppComponentLabel(compName). AddAppManagedByLabel(). diff --git a/controllers/apps/configuration/policy_util_test.go b/controllers/apps/configuration/policy_util_test.go index d243009c070..451b92ac42b 100644 --- a/controllers/apps/configuration/policy_util_test.go +++ b/controllers/apps/configuration/policy_util_test.go @@ -42,14 +42,14 @@ import ( var ( defaultNamespace = "default" - itsSchemaKind = workloads.GroupVersion.WithKind(workloads.Kind) + itsSchemaKind = workloads.GroupVersion.WithKind(workloads.InstanceSetKind) ) func newMockInstanceSet(replicas int, name string, labels map[string]string) workloads.InstanceSet { uid, _ := password.Generate(12, 12, 0, true, false) return workloads.InstanceSet{ TypeMeta: metav1.TypeMeta{ - Kind: workloads.Kind, + Kind: workloads.InstanceSetKind, APIVersion: workloads.GroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ diff --git a/controllers/apps/transform_utils.go b/controllers/apps/transform_utils.go index 23c7f405550..859916a8c25 100644 --- a/controllers/apps/transform_utils.go +++ b/controllers/apps/transform_utils.go @@ -203,7 +203,7 @@ func isOwnedByComp(obj client.Object) bool { // isOwnedByInstanceSet is used to judge if the obj is owned by the InstanceSet controller func isOwnedByInstanceSet(obj client.Object) bool { for _, ref := range obj.GetOwnerReferences() { - if ref.Kind == workloads.Kind && ref.Controller != nil && *ref.Controller { + if ref.Kind == workloads.InstanceSetKind && ref.Controller != nil && *ref.Controller { return true } } diff --git a/controllers/apps/transform_utils_test.go b/controllers/apps/transform_utils_test.go index 5033992fec4..2e5e34e583a 100644 --- a/controllers/apps/transform_utils_test.go +++ b/controllers/apps/transform_utils_test.go @@ -100,7 +100,7 @@ func TestIsOwnedByInstanceSet(t *testing.T) { its.OwnerReferences = []metav1.OwnerReference{ { - Kind: workloads.Kind, + Kind: workloads.InstanceSetKind, Controller: pointer.Bool(true), }, } diff --git a/controllers/apps/transformer_component_status.go b/controllers/apps/transformer_component_status.go index ea31b56d8d2..16761075d9e 100644 --- a/controllers/apps/transformer_component_status.go +++ b/controllers/apps/transformer_component_status.go @@ -290,7 +290,7 @@ func (t *componentStatusTransformer) hasFailedPod() (bool, appsv1alpha1.Componen hasFailedPod := meta.IsStatusConditionTrue(t.runningITS.Status.Conditions, string(workloads.InstanceFailure)) if hasFailedPod { failureCondition := meta.FindStatusCondition(t.runningITS.Status.Conditions, string(workloads.InstanceFailure)) - messages.SetObjectMessage(workloads.Kind, t.runningITS.Name, failureCondition.Message) + messages.SetObjectMessage(workloads.InstanceSetKind, t.runningITS.Name, failureCondition.Message) return true, messages } @@ -309,7 +309,7 @@ func (t *componentStatusTransformer) hasFailedPod() (bool, appsv1alpha1.Componen probeTimeoutDuration := time.Duration(defaultRoleProbeTimeoutAfterPodsReady) * time.Second condition := meta.FindStatusCondition(t.runningITS.Status.Conditions, string(workloads.InstanceReady)) if time.Now().After(condition.LastTransitionTime.Add(probeTimeoutDuration)) { - messages.SetObjectMessage(workloads.Kind, t.runningITS.Name, "Role probe timeout, check whether the application is available") + messages.SetObjectMessage(workloads.InstanceSetKind, t.runningITS.Name, "Role probe timeout, check whether the application is available") return true, messages } diff --git a/controllers/trace/change_capture_store.go b/controllers/trace/change_capture_store.go new file mode 100644 index 00000000000..fdd8af2e61c --- /dev/null +++ b/controllers/trace/change_capture_store.go @@ -0,0 +1,191 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "sort" + "strconv" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/controller/model" +) + +type ChangeCaptureStore interface { + Load(objects ...client.Object) error + Insert(object client.Object) error + Update(object client.Object) error + Delete(object client.Object) error + Get(objectRef *model.GVKNObjKey) client.Object + List(gvk *schema.GroupVersionKind) []client.Object + GetAll() map[model.GVKNObjKey]client.Object + GetChanges() []tracev1.ObjectChange +} + +type changeCaptureStore struct { + scheme *runtime.Scheme + formatter descriptionFormatter + store map[model.GVKNObjKey]client.Object + clock int64 + changes []tracev1.ObjectChange +} + +func (s *changeCaptureStore) Load(objects ...client.Object) error { + for _, object := range objects { + // sync the clock + revision := parseRevision(object.GetResourceVersion()) + if revision > s.clock { + s.clock = revision + } + objectRef, err := getObjectRef(object, s.scheme) + if err != nil { + return err + } + s.store[*objectRef] = object + + } + return nil +} + +func (s *changeCaptureStore) Insert(object client.Object) error { + var err error + obj := object.DeepCopyObject().(client.Object) + obj, err = normalize(obj) + if err != nil { + return err + } + objectRef, err := getObjectRef(obj, s.scheme) + if err != nil { + return err + } + obj.SetResourceVersion(s.applyRevision()) + object.SetResourceVersion(obj.GetResourceVersion()) + s.store[*objectRef] = obj + + s.captureCreation(objectRef, obj) + + return nil +} + +func (s *changeCaptureStore) Update(object client.Object) error { + var err error + newObj := object.DeepCopyObject().(client.Object) + newObj, err = normalize(newObj) + if err != nil { + return err + } + objectRef, err := getObjectRef(newObj, s.scheme) + if err != nil { + return err + } + oldObj := s.store[*objectRef] + newObj.SetResourceVersion(s.applyRevision()) + object.SetResourceVersion(newObj.GetResourceVersion()) + s.store[*objectRef] = newObj + + s.captureUpdate(objectRef, oldObj, newObj) + return nil +} + +func (s *changeCaptureStore) Delete(object client.Object) error { + objectRef, err := getObjectRef(object, s.scheme) + if err != nil { + return err + } + obj, ok := s.store[*objectRef] + if !ok { + return nil + } + delete(s.store, *objectRef) + + s.captureDeletion(objectRef, obj) + return nil +} + +func (s *changeCaptureStore) Get(objectRef *model.GVKNObjKey) client.Object { + return s.store[*objectRef] +} + +func (s *changeCaptureStore) List(gvk *schema.GroupVersionKind) []client.Object { + var objects []client.Object + for objectRef, object := range s.store { + if objectRef.GroupVersionKind == *gvk { + objects = append(objects, object) + } + } + return objects +} + +func (s *changeCaptureStore) GetAll() map[model.GVKNObjKey]client.Object { + all := make(map[model.GVKNObjKey]client.Object, len(s.store)) + for key, object := range s.store { + all[key] = object.DeepCopyObject().(client.Object) + } + return all +} + +func (s *changeCaptureStore) GetChanges() []tracev1.ObjectChange { + sort.SliceStable(s.changes, func(i, j int) bool { + return s.changes[i].Revision < s.changes[j].Revision + }) + return s.changes +} + +func newChangeCaptureStore(scheme *runtime.Scheme, formatter descriptionFormatter) ChangeCaptureStore { + return &changeCaptureStore{ + scheme: scheme, + store: make(map[model.GVKNObjKey]client.Object), + formatter: formatter, + } +} + +func (s *changeCaptureStore) applyRevision() string { + s.clock++ + return strconv.FormatInt(s.clock, 10) +} + +func (s *changeCaptureStore) captureCreation(objectRef *model.GVKNObjKey, object client.Object) { + changes := buildChanges( + make(map[model.GVKNObjKey]client.Object), + map[model.GVKNObjKey]client.Object{*objectRef: object}, + s.formatter) + s.changes = append(s.changes, changes...) +} + +func (s *changeCaptureStore) captureUpdate(objectRef *model.GVKNObjKey, obj client.Object, object client.Object) { + changes := buildChanges( + map[model.GVKNObjKey]client.Object{*objectRef: obj}, + map[model.GVKNObjKey]client.Object{*objectRef: object}, + s.formatter) + s.changes = append(s.changes, changes...) +} + +func (s *changeCaptureStore) captureDeletion(objectRef *model.GVKNObjKey, object client.Object) { + changes := buildChanges( + map[model.GVKNObjKey]client.Object{*objectRef: object}, + make(map[model.GVKNObjKey]client.Object), + s.formatter) + s.changes = append(s.changes, changes...) +} + +var _ ChangeCaptureStore = &changeCaptureStore{} diff --git a/controllers/trace/change_capture_store_test.go b/controllers/trace/change_capture_store_test.go new file mode 100644 index 00000000000..34e4f2777ae --- /dev/null +++ b/controllers/trace/change_capture_store_test.go @@ -0,0 +1,95 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/client-go/kubernetes/scheme" + + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/controller/builder" +) + +var _ = Describe("change_capture_store test", func() { + Context("Testing change_capture_store", func() { + It("should work well", func() { + i18n := builder.NewConfigMapBuilder(namespace, name).SetData( + map[string]string{"en": "apps.kubeblocks.io/v1/Component/Creation=Component %s/%s is created."}, + ).GetObject() + store := newChangeCaptureStore(scheme.Scheme, buildDescriptionFormatter(i18n, defaultLocale, nil)) + + By("Load a cluster") + primary := builder.NewClusterBuilder(namespace, name).SetUID(uid).SetResourceVersion(resourceVersion).GetObject() + Expect(store.Load(primary)).Should(Succeed()) + primaryRef, err := getObjectRef(primary, scheme.Scheme) + Expect(err).Should(BeNil()) + Expect(store.Get(primaryRef)).ShouldNot(BeNil()) + + By("Insert a component") + compName := "test" + fullCompName := fmt.Sprintf("%s-%s", primary.Name, compName) + secondary := builder.NewComponentBuilder(namespace, fullCompName, ""). + SetOwnerReferences(kbappsv1.APIVersion, kbappsv1.ClusterKind, primary). + SetUID(uid). + GetObject() + secondary.ResourceVersion = resourceVersion + Expect(store.Insert(secondary)).Should(Succeed()) + objectRef, err := getObjectRef(secondary, scheme.Scheme) + Expect(err).Should(BeNil()) + Expect(store.Get(objectRef)).ShouldNot(BeNil()) + + By("Update the component") + secondary.ResourceVersion = "123456" + Expect(store.Update(secondary)).Should(Succeed()) + Expect(store.Get(objectRef)).Should(Equal(secondary)) + + By("List all components") + objects := store.List(&objectRef.GroupVersionKind) + Expect(objects).Should(HaveLen(1)) + Expect(objects[0]).Should(Equal(secondary)) + + By("GetAll components") + objectMap := store.GetAll() + Expect(objectMap).Should(HaveLen(2)) + v, ok := objectMap[*primaryRef] + Expect(ok).Should(BeTrue()) + Expect(v).Should(Equal(primary)) + v, ok = objectMap[*objectRef] + Expect(ok).Should(BeTrue()) + Expect(v).Should(Equal(secondary)) + + By("Delete the component") + Expect(store.Delete(secondary)).Should(Succeed()) + Expect(store.Get(objectRef)).Should(BeNil()) + + By("GetChanges") + changes := store.GetChanges() + Expect(changes).Should(HaveLen(3)) + Expect(changes[0].ChangeType).Should(Equal(tracev1.ObjectCreationType)) + Expect(changes[1].ChangeType).Should(Equal(tracev1.ObjectUpdateType)) + Expect(changes[2].ChangeType).Should(Equal(tracev1.ObjectDeletionType)) + }) + }) +}) diff --git a/controllers/trace/current_state_handler.go b/controllers/trace/current_state_handler.go new file mode 100644 index 00000000000..8d49c1bdf45 --- /dev/null +++ b/controllers/trace/current_state_handler.go @@ -0,0 +1,202 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + "sort" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx" + "github.com/apecloud/kubeblocks/pkg/controller/model" +) + +type traceCalculator struct { + ctx context.Context + cli client.Client + scheme *runtime.Scheme + store ObjectRevisionStore +} + +func (c *traceCalculator) PreCondition(tree *kubebuilderx.ObjectTree) *kubebuilderx.CheckResult { + if tree.GetRoot() == nil || model.IsObjectDeleting(tree.GetRoot()) { + return kubebuilderx.ConditionUnsatisfied + } + return kubebuilderx.ConditionSatisfied +} + +func (c *traceCalculator) Reconcile(tree *kubebuilderx.ObjectTree) (kubebuilderx.Result, error) { + trace, _ := tree.GetRoot().(*tracev1.ReconciliationTrace) + objs := tree.List(&corev1.ConfigMap{}) + var i18nResource *corev1.ConfigMap + if len(objs) > 0 { + i18nResource, _ = objs[0].(*corev1.ConfigMap) + } + + root := &kbappsv1.Cluster{} + objectKey := client.ObjectKeyFromObject(trace) + if trace.Spec.TargetObject != nil { + objectKey = client.ObjectKey{ + Namespace: trace.Spec.TargetObject.Namespace, + Name: trace.Spec.TargetObject.Name, + } + } + if err := c.cli.Get(c.ctx, objectKey, root); err != nil { + return kubebuilderx.Commit, err + } + + // handle object changes + // build new object set from cache + newObjectMap, err := getObjectsFromCache(c.ctx, c.cli, root, getKBOwnershipRules()) + if err != nil { + return kubebuilderx.Commit, err + } + // build old object set from store + currentState := &trace.Status.CurrentState + oldObjectMap, err := getObjectsFromTree(currentState.ObjectTree, c.store, c.scheme) + if err != nil { + return kubebuilderx.Commit, err + } + changes := buildChanges(oldObjectMap, newObjectMap, buildDescriptionFormatter(i18nResource, defaultLocale, trace.Spec.Locale)) + + // handle event changes + newEventMap, err := filterEvents(getEventsFromCache(c.ctx, c.cli), newObjectMap) + if err != nil { + return kubebuilderx.Commit, err + } + oldEventMap, err := filterEvents(getEventsFromStore(c.store), oldObjectMap) + if err != nil { + return kubebuilderx.Commit, err + } + eventChanges := buildChanges(oldEventMap, newEventMap, buildDescriptionFormatter(i18nResource, defaultLocale, trace.Spec.Locale)) + changes = append(changes, eventChanges...) + + if len(changes) == 0 { + return kubebuilderx.Continue, nil + } + + // sort the changes by resource version. + sort.SliceStable(changes, func(i, j int) bool { + return changes[i].Revision < changes[j].Revision + }) + + // concat it to current changes + currentState.Changes = append(currentState.Changes, changes...) + + // save new version objects to store + for _, object := range newObjectMap { + if err = c.store.Insert(object, trace); err != nil { + return kubebuilderx.Commit, err + } + } + // save new events to store + for _, object := range newEventMap { + if err = c.store.Insert(object, trace); err != nil { + return kubebuilderx.Commit, err + } + } + + // update current object tree + currentState.ObjectTree, err = getObjectTreeWithRevision(root, getKBOwnershipRules(), c.store, currentState.Changes[len(currentState.Changes)-1].Revision, c.scheme) + if err != nil { + return kubebuilderx.Commit, err + } + + // update changes summary + initialObjectMap, err := getObjectsFromTree(trace.Status.InitialObjectTree, c.store, c.scheme) + if err != nil { + return kubebuilderx.Commit, err + } + + currentState.Summary.ObjectSummaries = buildObjectSummaries(initialObjectMap, newObjectMap) + + return kubebuilderx.Continue, nil +} + +func getEventsFromCache(ctx context.Context, cli client.Client) func() ([]client.Object, error) { + return func() ([]client.Object, error) { + eventList := &corev1.EventList{} + if err := cli.List(ctx, eventList); err != nil { + return nil, err + } + var objects []client.Object + for i := range eventList.Items { + objects = append(objects, &eventList.Items[i]) + } + return objects, nil + } +} + +func getEventsFromStore(store ObjectRevisionStore) func() ([]client.Object, error) { + return func() ([]client.Object, error) { + eventRevisionMap := store.List(&eventGVK) + var objects []client.Object + for _, revisionMap := range eventRevisionMap { + revision := int64(-1) + for rev := range revisionMap { + if rev > revision { + revision = rev + } + } + if revision > -1 { + objects = append(objects, revisionMap[revision]) + } + } + return objects, nil + } +} + +func filterEvents(eventLister func() ([]client.Object, error), objectMap map[model.GVKNObjKey]client.Object) (map[model.GVKNObjKey]client.Object, error) { + eventList, err := eventLister() + if err != nil { + return nil, err + } + objectRefSet := sets.KeySet(objectMap) + matchedEventMap := make(map[model.GVKNObjKey]client.Object) + for i := range eventList { + event, _ := eventList[i].(*corev1.Event) + objRef := objectReferenceToRef(&event.InvolvedObject) + if objectRefSet.Has(*objRef) { + eventRef := model.GVKNObjKey{ + GroupVersionKind: eventGVK, + ObjectKey: client.ObjectKeyFromObject(event), + } + matchedEventMap[eventRef] = event + } + } + return matchedEventMap, nil +} + +func updateCurrentState(ctx context.Context, cli client.Client, scheme *runtime.Scheme, store ObjectRevisionStore) kubebuilderx.Reconciler { + return &traceCalculator{ + ctx: ctx, + cli: cli, + scheme: scheme, + store: store, + } +} + +var _ kubebuilderx.Reconciler = &traceCalculator{} diff --git a/controllers/trace/current_state_handler_test.go b/controllers/trace/current_state_handler_test.go new file mode 100644 index 00000000000..03a1490c21b --- /dev/null +++ b/controllers/trace/current_state_handler_test.go @@ -0,0 +1,100 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/golang/mock/gomock" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/controller/builder" + "github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx" + testutil "github.com/apecloud/kubeblocks/pkg/testutil/k8s" + "github.com/apecloud/kubeblocks/pkg/testutil/k8s/mocks" +) + +var _ = Describe("current_state_handler test", func() { + var ( + k8sMock *mocks.MockClient + controller *gomock.Controller + ) + + BeforeEach(func() { + controller, k8sMock = testutil.SetupK8sMock() + }) + + AfterEach(func() { + controller.Finish() + }) + + Context("Testing current_state_handler", func() { + It("should work well", func() { + store := NewObjectStore(scheme.Scheme) + reconciler := updateCurrentState(ctx, k8sMock, scheme.Scheme, store) + + primary, _ := mockObjects(k8sMock) + trace := &tracev1.ReconciliationTrace{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: tracev1.ReconciliationTraceSpec{ + TargetObject: &tracev1.ObjectReference{ + Namespace: primary.Namespace, + Name: primary.Name, + }, + }, + } + tree := kubebuilderx.NewObjectTree() + tree.SetRoot(trace) + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &kbappsv1.Cluster{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *kbappsv1.Cluster, _ ...client.GetOption) error { + *obj = *primary + return nil + }) + objectRef, err := getObjectReference(primary, scheme.Scheme) + Expect(err).ToNot(HaveOccurred()) + event := builder.NewEventBuilder(namespace, name).SetInvolvedObject(*objectRef).GetObject() + k8sMock.EXPECT(). + List(gomock.Any(), &corev1.EventList{}, gomock.Any()). + DoAndReturn(func(_ context.Context, list *corev1.EventList, _ ...client.ListOption) error { + list.Items = []corev1.Event{*event} + return nil + }) + + res, err := reconciler.Reconcile(tree) + Expect(err).ToNot(HaveOccurred()) + Expect(res).Should(Equal(kubebuilderx.Continue)) + Expect(trace.Status.CurrentState.ObjectTree).ShouldNot(BeNil()) + Expect(trace.Status.CurrentState.Changes).Should(HaveLen(4)) + Expect(trace.Status.CurrentState.Summary.ObjectSummaries).Should(HaveLen(2)) + }) + }) +}) diff --git a/controllers/trace/deletion_handler.go b/controllers/trace/deletion_handler.go new file mode 100644 index 00000000000..c5f4bf1a588 --- /dev/null +++ b/controllers/trace/deletion_handler.go @@ -0,0 +1,55 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx" + "github.com/apecloud/kubeblocks/pkg/controller/model" +) + +type deletionHandler struct { + store ObjectRevisionStore +} + +func (h *deletionHandler) PreCondition(tree *kubebuilderx.ObjectTree) *kubebuilderx.CheckResult { + if tree.GetRoot() == nil || !model.IsObjectDeleting(tree.GetRoot()) { + return kubebuilderx.ConditionUnsatisfied + } + return kubebuilderx.ConditionSatisfied +} + +func (h *deletionHandler) Reconcile(tree *kubebuilderx.ObjectTree) (kubebuilderx.Result, error) { + trace, _ := tree.GetRoot().(*tracev1.ReconciliationTrace) + + // store cleanup + deleteUnusedRevisions(h.store, trace.Status.CurrentState.Changes, trace) + + // remove finalizer + tree.DeleteRoot() + + return kubebuilderx.Commit, nil +} + +func handleDeletion(store ObjectRevisionStore) kubebuilderx.Reconciler { + return &deletionHandler{store: store} +} + +var _ kubebuilderx.Reconciler = &deletionHandler{} diff --git a/controllers/trace/deletion_handler_test.go b/controllers/trace/deletion_handler_test.go new file mode 100644 index 00000000000..2095b371fc6 --- /dev/null +++ b/controllers/trace/deletion_handler_test.go @@ -0,0 +1,60 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx" +) + +var _ = Describe("deletion_handler test", func() { + Context("Testing deletion_handler", func() { + It("should work well", func() { + store := NewObjectStore(scheme.Scheme) + reconciler := handleDeletion(store) + + trace := &tracev1.ReconciliationTrace{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + } + tree := kubebuilderx.NewObjectTree() + tree.SetRoot(trace) + + Expect(reconciler.PreCondition(tree)).To(Equal(kubebuilderx.ConditionUnsatisfied)) + trace.DeletionTimestamp = &metav1.Time{Time: time.Now()} + Expect(reconciler.PreCondition(tree)).To(Equal(kubebuilderx.ConditionSatisfied)) + + res, err := reconciler.Reconcile(tree) + Expect(err).ToNot(HaveOccurred()) + Expect(res).Should(Equal(kubebuilderx.Commit)) + Expect(tree.GetRoot()).Should(BeNil()) + }) + }) +}) diff --git a/controllers/trace/desired_state_handler.go b/controllers/trace/desired_state_handler.go new file mode 100644 index 00000000000..37c2b39c5d5 --- /dev/null +++ b/controllers/trace/desired_state_handler.go @@ -0,0 +1,230 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + "fmt" + + "github.com/google/cel-go/cel" + "github.com/google/cel-go/checker/decls" + "github.com/google/cel-go/common/types" + "google.golang.org/protobuf/types/known/structpb" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx" + "github.com/apecloud/kubeblocks/pkg/controller/model" +) + +type stateEvaluation struct { + ctx context.Context + cli client.Client + store ObjectRevisionStore + scheme *runtime.Scheme +} + +func (s *stateEvaluation) PreCondition(tree *kubebuilderx.ObjectTree) *kubebuilderx.CheckResult { + if tree.GetRoot() == nil || model.IsObjectDeleting(tree.GetRoot()) { + return kubebuilderx.ConditionUnsatisfied + } + return kubebuilderx.ConditionSatisfied +} + +func (s *stateEvaluation) Reconcile(tree *kubebuilderx.ObjectTree) (kubebuilderx.Result, error) { + trace, _ := tree.GetRoot().(*tracev1.ReconciliationTrace) + objs := tree.List(&corev1.ConfigMap{}) + var i18nResource *corev1.ConfigMap + if len(objs) > 0 { + i18nResource, _ = objs[0].(*corev1.ConfigMap) + } + + // build new object set from cache + root := &kbappsv1.Cluster{} + objectKey := client.ObjectKeyFromObject(trace) + if trace.Spec.TargetObject != nil { + objectKey = client.ObjectKey{ + Namespace: trace.Spec.TargetObject.Namespace, + Name: trace.Spec.TargetObject.Name, + } + } + if err := s.cli.Get(s.ctx, objectKey, root); err != nil { + return kubebuilderx.Commit, err + } + + // keep only the latest reconciliation cycle. + // the basic idea is: + // 1. traverse the trace progress list from tail to head, + // 2. read the corresponding version of the Cluster object from object store by objectChange.resourceVersion, + // 3. evaluation its state, + // if we find the first-false-then-true pattern, means a new reconciliation cycle starts. + firstFalseStateFound := false + clusterType := tracev1.ObjectType{ + APIVersion: kbappsv1.APIVersion, + Kind: kbappsv1.ClusterKind, + } + latestReconciliationCycleStart := -1 + var initialRoot *kbappsv1.Cluster + for i := len(trace.Status.CurrentState.Changes) - 1; i >= 0; i-- { + change := trace.Status.CurrentState.Changes[i] + objType := objectReferenceToType(&change.ObjectReference) + if *objType != clusterType { + continue + } + objectRef := objectReferenceToRef(&change.ObjectReference) + obj, err := s.store.Get(objectRef, change.Revision) + if err != nil && !apierrors.IsNotFound(err) { + return kubebuilderx.Commit, err + } + // handle revision lost after controller restarted + if obj == nil { + continue + } + expr := defaultStateEvaluationExpression + if trace.Spec.StateEvaluationExpression != nil { + expr = *trace.Spec.StateEvaluationExpression + } + state, err := doStateEvaluation(obj, expr) + if err != nil { + return kubebuilderx.Commit, err + } + if !state && !firstFalseStateFound { + firstFalseStateFound = true + } + if state && firstFalseStateFound { + latestReconciliationCycleStart = i + initialRoot, _ = obj.(*kbappsv1.Cluster) + break + } + } + if latestReconciliationCycleStart <= 0 { + if trace.Status.InitialObjectTree == nil { + reference, err := getObjectReference(root, s.scheme) + if err != nil { + return kubebuilderx.Commit, err + } + trace.Status.InitialObjectTree = &tracev1.ObjectTreeNode{ + Primary: *reference, + } + } + return kubebuilderx.Continue, nil + } + + // build new InitialObjectTree + var err error + trace.Status.InitialObjectTree, err = getObjectTreeWithRevision(initialRoot, getKBOwnershipRules(), s.store, trace.Status.CurrentState.Changes[latestReconciliationCycleStart].Revision, s.scheme) + if err != nil { + return kubebuilderx.Commit, err + } + + // update desired state + generator := newPlanGenerator(s.ctx, s.cli, s.scheme, + treeObjectLoader(trace.Status.InitialObjectTree, s.store, s.scheme), + buildDescriptionFormatter(i18nResource, defaultLocale, trace.Spec.Locale)) + plan, err := generator.generatePlan(root) + if err != nil { + return kubebuilderx.Commit, err + } + trace.Status.DesiredState = &plan.Plan + + // delete unused object revisions + deleteUnusedRevisions(s.store, trace.Status.CurrentState.Changes[:latestReconciliationCycleStart], trace) + + // truncate outage changes + trace.Status.CurrentState.Changes = trace.Status.CurrentState.Changes[latestReconciliationCycleStart:] + + return kubebuilderx.Continue, nil +} + +func updateDesiredState(ctx context.Context, cli client.Client, scheme *runtime.Scheme, store ObjectRevisionStore) kubebuilderx.Reconciler { + return &stateEvaluation{ + ctx: ctx, + cli: cli, + scheme: scheme, + store: store, + } +} + +func doStateEvaluation(object client.Object, expression tracev1.StateEvaluationExpression) (bool, error) { + if expression.CELExpression == nil { + return false, fmt.Errorf("CEL expression can't be empty") + } + + // Convert the object to an unstructured map + unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(object) + if err != nil { + return false, fmt.Errorf("failed to convert object to unstructured: %w", err) + } + + // Create a CEL environment with the object fields available + env, err := cel.NewEnv( + cel.Declarations( + decls.NewVar("object", decls.NewMapType(decls.String, decls.Dyn)), + ), + ) + if err != nil { + return false, fmt.Errorf("failed to create CEL environment: %w", err) + } + + // Compile the CEL expression + ast, issues := env.Compile(expression.CELExpression.Expression) + if issues.Err() != nil { + return false, fmt.Errorf("failed to compile expression: %w", issues.Err()) + } + + // Create a program + prg, err := env.Program(ast) + if err != nil { + return false, fmt.Errorf("failed to create CEL program: %w", err) + } + + // Evaluate the expression with the object map + objValue, err := structpb.NewStruct(unstructuredMap) + if err != nil { + return false, fmt.Errorf("failed to create structpb value: %w", err) + } + + out, _, err := prg.Eval(map[string]any{ + "object": types.NewDynamicMap(types.DefaultTypeAdapter, objValue.AsMap()), + }) + if err != nil { + return false, fmt.Errorf("failed to evaluate expression: %w", err) + } + + // Convert the output to a boolean + result, ok := out.Value().(bool) + if !ok { + return false, fmt.Errorf("expression did not return a boolean") + } + + return result, nil +} + +func treeObjectLoader(tree *tracev1.ObjectTreeNode, store ObjectRevisionStore, scheme *runtime.Scheme) objectLoader { + return func() (map[model.GVKNObjKey]client.Object, error) { + return getObjectsFromTree(tree, store, scheme) + } +} + +var _ kubebuilderx.Reconciler = &stateEvaluation{} diff --git a/controllers/trace/desired_state_handler_test.go b/controllers/trace/desired_state_handler_test.go new file mode 100644 index 00000000000..44f24b36b3b --- /dev/null +++ b/controllers/trace/desired_state_handler_test.go @@ -0,0 +1,305 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/golang/mock/gomock" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/constant" + "github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx" + testutil "github.com/apecloud/kubeblocks/pkg/testutil/k8s" + "github.com/apecloud/kubeblocks/pkg/testutil/k8s/mocks" +) + +var _ = Describe("desired_state_handler test", func() { + var ( + k8sMock *mocks.MockClient + controller *gomock.Controller + ) + + BeforeEach(func() { + controller, k8sMock = testutil.SetupK8sMock() + }) + + AfterEach(func() { + controller.Finish() + }) + + Context("Testing desired_state_handler", func() { + It("should work well", func() { + clusterDefinition := &kbappsv1.ClusterDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + Generation: int64(1), + }, + Spec: kbappsv1.ClusterDefinitionSpec{ + Topologies: []kbappsv1.ClusterTopology{ + { + Name: name, + Default: true, + Components: []kbappsv1.ClusterTopologyComponent{ + { + Name: name, + CompDef: name, + }, + }, + }, + }, + }, + Status: kbappsv1.ClusterDefinitionStatus{ + ObservedGeneration: int64(1), + Phase: kbappsv1.AvailablePhase, + }, + } + serviceVersion := "1.0.0" + componentDefinition := &kbappsv1.ComponentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + Generation: int64(1), + }, + Spec: kbappsv1.ComponentDefinitionSpec{ + ServiceVersion: serviceVersion, + Runtime: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: name, + Image: "busybox", + }}, + }, + }, + Status: kbappsv1.ComponentDefinitionStatus{ + ObservedGeneration: int64(1), + Phase: kbappsv1.AvailablePhase, + }, + } + componentVersion := &kbappsv1.ComponentVersion{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + Generation: int64(1), + Annotations: map[string]string{ + "componentversion.kubeblocks.io/compatible-definitions": name, + }, + }, + Spec: kbappsv1.ComponentVersionSpec{ + CompatibilityRules: []kbappsv1.ComponentVersionCompatibilityRule{{ + CompDefs: []string{name}, + Releases: []string{name}, + }}, + Releases: []kbappsv1.ComponentVersionRelease{{ + Name: name, + ServiceVersion: serviceVersion, + Images: map[string]string{ + name: "busybox", + }, + }}, + }, + Status: kbappsv1.ComponentVersionStatus{ + ObservedGeneration: int64(1), + Phase: kbappsv1.AvailablePhase, + }, + } + clusterTemplate := &kbappsv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + UID: uid, + ResourceVersion: "1", + }, + Spec: kbappsv1.ClusterSpec{ + ClusterDef: name, + TerminationPolicy: kbappsv1.WipeOut, + ComponentSpecs: []kbappsv1.ClusterComponentSpec{{ + Name: name, + Replicas: 0, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + VolumeClaimTemplates: []kbappsv1.ClusterComponentVolumeClaimTemplate{{ + Name: name, + Spec: kbappsv1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("20Gi"), + }, + }, + }, + }}, + }}, + }, + } + primaryV1 := clusterTemplate.DeepCopy() + primaryV1.Generation = 1 + primaryV1.ResourceVersion = "2" + primaryV1.Status.Phase = kbappsv1.RunningClusterPhase + primaryV2 := clusterTemplate.DeepCopy() + primaryV2.Generation = 2 + primaryV2.ResourceVersion = "3" + primaryV2.Spec.ComponentSpecs[0].Replicas = 1 + ref0, err := getObjectReference(clusterTemplate, scheme.Scheme) + Expect(err).ToNot(HaveOccurred()) + ref1, err := getObjectReference(primaryV1, scheme.Scheme) + Expect(err).ToNot(HaveOccurred()) + ref2, err := getObjectReference(primaryV2, scheme.Scheme) + Expect(err).ToNot(HaveOccurred()) + trace := &tracev1.ReconciliationTrace{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: tracev1.ReconciliationTraceSpec{ + TargetObject: &tracev1.ObjectReference{ + Namespace: primaryV1.Namespace, + Name: primaryV1.Name, + }, + }, + Status: tracev1.ReconciliationTraceStatus{ + CurrentState: tracev1.ReconciliationCycleState{ + Changes: []tracev1.ObjectChange{ + { + ObjectReference: *ref0, + ChangeType: tracev1.ObjectCreationType, + Revision: parseRevision(ref0.ResourceVersion), + Description: "Creation", + }, + { + ObjectReference: *ref1, + ChangeType: tracev1.ObjectUpdateType, + Revision: parseRevision(ref1.ResourceVersion), + Description: "Update", + }, + { + ObjectReference: *ref2, + ChangeType: tracev1.ObjectUpdateType, + Revision: parseRevision(ref2.ResourceVersion), + Description: "Update", + }, + }, + }, + }, + } + + store := NewObjectStore(scheme.Scheme) + Expect(store.Insert(primaryV1, trace)).Should(Succeed()) + Expect(store.Insert(primaryV2, trace)).Should(Succeed()) + + tree := kubebuilderx.NewObjectTree() + tree.SetRoot(trace) + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &kbappsv1.ClusterDefinition{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *kbappsv1.ClusterDefinition, _ ...client.GetOption) error { + *obj = *clusterDefinition.DeepCopy() + return nil + }).AnyTimes() + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &kbappsv1.ComponentDefinition{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *kbappsv1.ComponentDefinition, _ ...client.GetOption) error { + *obj = *componentDefinition.DeepCopy() + return nil + }).AnyTimes() + k8sMock.EXPECT(). + List(gomock.Any(), &kbappsv1.ComponentDefinitionList{}, gomock.Any()). + DoAndReturn(func(_ context.Context, list *kbappsv1.ComponentDefinitionList, _ ...client.GetOption) error { + list.Items = []kbappsv1.ComponentDefinition{*componentDefinition.DeepCopy()} + return nil + }).AnyTimes() + k8sMock.EXPECT(). + List(gomock.Any(), &kbappsv1.ComponentVersionList{}, gomock.Any()). + DoAndReturn(func(_ context.Context, list *kbappsv1.ComponentVersionList, _ ...client.ListOption) error { + list.Items = []kbappsv1.ComponentVersion{*componentVersion.DeepCopy()} + return nil + }).AnyTimes() + k8sMock.EXPECT(). + List(gomock.Any(), &kbappsv1.SidecarDefinitionList{}, gomock.Any()). + DoAndReturn(func(_ context.Context, list *kbappsv1.SidecarDefinitionList, _ ...client.ListOption) error { + return nil + }).AnyTimes() + k8sMock.EXPECT(). + List(gomock.Any(), &dpv1alpha1.BackupPolicyTemplateList{}, gomock.Any()). + DoAndReturn(func(_ context.Context, list *dpv1alpha1.BackupPolicyTemplateList, _ ...client.ListOption) error { + return nil + }).AnyTimes() + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &kbappsv1.Cluster{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *kbappsv1.Cluster, _ ...client.GetOption) error { + *obj = *primaryV2.DeepCopy() + return nil + }).Times(1) + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &kbappsv1.Component{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *kbappsv1.Component, _ ...client.GetOption) error { + return apierrors.NewNotFound(kbappsv1.Resource(kbappsv1.ComponentKind), objKey.Name) + }).AnyTimes() + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &appsv1alpha1.Configuration{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *appsv1alpha1.Configuration, _ ...client.GetOption) error { + return apierrors.NewNotFound(appsv1alpha1.Resource(constant.ConfigurationKind), objKey.Name) + }).AnyTimes() + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &corev1.ConfigMap{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *corev1.ConfigMap, _ ...client.GetOption) error { + return apierrors.NewNotFound(corev1.Resource(constant.ConfigMapKind), objKey.Name) + }).AnyTimes() + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &corev1.PersistentVolumeClaim{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *corev1.PersistentVolumeClaim, _ ...client.GetOption) error { + return apierrors.NewNotFound(corev1.Resource(constant.PersistentVolumeClaimKind), objKey.Name) + }).AnyTimes() + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &corev1.PersistentVolume{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *corev1.PersistentVolume, _ ...client.GetOption) error { + return apierrors.NewNotFound(corev1.Resource(constant.PersistentVolumeKind), objKey.Name) + }).AnyTimes() + k8sMock.EXPECT().Scheme().Return(scheme.Scheme).AnyTimes() + + reconciler := updateDesiredState(ctx, k8sMock, scheme.Scheme, store) + res, err := reconciler.Reconcile(tree) + Expect(err).ToNot(HaveOccurred()) + Expect(res).Should(Equal(kubebuilderx.Continue)) + Expect(trace.Status.CurrentState.Changes).Should(HaveLen(2)) + Expect(trace.Status.DesiredState.ObjectTree).ShouldNot(BeNil()) + Expect(len(trace.Status.DesiredState.Changes)).Should(BeNumerically(">", 0)) + Expect(trace.Status.DesiredState.Summary.ObjectSummaries).ShouldNot(BeNil()) + }) + }) +}) diff --git a/controllers/trace/dry_run_handler.go b/controllers/trace/dry_run_handler.go new file mode 100644 index 00000000000..3b4bacd50bf --- /dev/null +++ b/controllers/trace/dry_run_handler.go @@ -0,0 +1,185 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + "encoding/json" + "fmt" + "hash/fnv" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/strategicpatch" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/constant" + "github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx" + "github.com/apecloud/kubeblocks/pkg/controller/model" +) + +type dryRunner struct { + ctx context.Context + cli client.Client + scheme *runtime.Scheme +} + +func (r *dryRunner) PreCondition(tree *kubebuilderx.ObjectTree) *kubebuilderx.CheckResult { + if tree.GetRoot() == nil || model.IsObjectDeleting(tree.GetRoot()) { + return kubebuilderx.ConditionUnsatisfied + } + v, _ := tree.GetRoot().(*tracev1.ReconciliationTrace) + if isDesiredSpecChanged(v) { + return kubebuilderx.ConditionSatisfied + } + return kubebuilderx.ConditionUnsatisfied +} + +func (r *dryRunner) Reconcile(tree *kubebuilderx.ObjectTree) (kubebuilderx.Result, error) { + trace, _ := tree.GetRoot().(*tracev1.ReconciliationTrace) + objs := tree.List(&corev1.ConfigMap{}) + var i18nResource *corev1.ConfigMap + if len(objs) > 0 { + i18nResource, _ = objs[0].(*corev1.ConfigMap) + } + + root := &kbappsv1.Cluster{} + objectKey := client.ObjectKeyFromObject(trace) + if trace.Spec.TargetObject != nil { + objectKey = client.ObjectKey{ + Namespace: trace.Spec.TargetObject.Namespace, + Name: trace.Spec.TargetObject.Name, + } + } + if err := r.cli.Get(r.ctx, objectKey, root); err != nil { + return kubebuilderx.Commit, err + } + + generator := newPlanGenerator(r.ctx, r.cli, r.scheme, + cacheObjectLoader(r.ctx, r.cli, root, getKBOwnershipRules()), + buildDescriptionFormatter(i18nResource, defaultLocale, trace.Spec.Locale)) + + desiredRoot, err := applySpec(root.DeepCopy(), trace.Spec.DryRun.DesiredSpec) + if err != nil { + return kubebuilderx.Commit, err + } + plan, err := generator.generatePlan(desiredRoot) + if err != nil { + return kubebuilderx.Commit, err + } + plan.DesiredSpecRevision = getDesiredSpecRevision(trace.Spec.DryRun.DesiredSpec) + trace.Status.DryRunResult = plan + + return kubebuilderx.Continue, nil +} + +func applySpec(current *kbappsv1.Cluster, desiredSpecStr string) (*kbappsv1.Cluster, error) { + // Convert the desiredSpec YAML string to a map + specMap := make(map[string]interface{}) + if err := yaml.Unmarshal([]byte(desiredSpecStr), &specMap); err != nil { + return nil, fmt.Errorf("failed to unmarshal desiredSpec: %w", err) + } + + // Extract the current spec and apply the patch + currentSpec, err := getFieldAsStruct(current, specFieldName) + if err != nil { + return nil, fmt.Errorf("failed to get current spec: %w", err) + } + + // Create a strategic merge patch + patchData, err := strategicpatch.CreateTwoWayMergePatch( + specMapToJSON(currentSpec), + specMapToJSON(specMap), + currentSpec, + ) + if err != nil { + return nil, err + } + + // Apply the patch to the current spec + modifiedSpec, err := strategicpatch.StrategicMergePatch( + specMapToJSON(currentSpec), + patchData, + currentSpec, + ) + if err != nil { + return nil, err + } + + modifiedSpecMap := make(map[string]interface{}) + if err = json.Unmarshal(modifiedSpec, &modifiedSpecMap); err != nil { + return nil, err + } + + // Convert the object to an unstructured map + objMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(current) + if err != nil { + return nil, err + } + + // Update the spec in the object map + if err := unstructured.SetNestedField(objMap, modifiedSpecMap, "spec"); err != nil { + return nil, err + } + + // Convert the modified map back to the original object + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(objMap, current); err != nil { + return nil, err + } + return current, nil +} + +func dryRun(ctx context.Context, cli client.Client, scheme *runtime.Scheme) kubebuilderx.Reconciler { + return &dryRunner{ + ctx: context.WithValue(ctx, constant.DryRunContextKey, true), + cli: cli, + scheme: scheme, + } +} + +func isDesiredSpecChanged(v *tracev1.ReconciliationTrace) bool { + if v.Spec.DryRun == nil && v.Status.DryRunResult == nil { + return false + } + if v.Spec.DryRun == nil || v.Status.DryRunResult == nil { + return true + } + revision := getDesiredSpecRevision(v.Spec.DryRun.DesiredSpec) + return revision != v.Status.DryRunResult.DesiredSpecRevision +} + +func getDesiredSpecRevision(desiredSpec string) string { + hf := fnv.New32() + _, _ = hf.Write([]byte(desiredSpec)) + return rand.SafeEncodeString(fmt.Sprint(hf.Sum32())) +} + +func cacheObjectLoader(ctx context.Context, cli client.Client, root *kbappsv1.Cluster, rules []OwnershipRule) objectLoader { + return func() (map[model.GVKNObjKey]client.Object, error) { + return getObjectsFromCache(ctx, cli, root, rules) + } +} + +var _ kubebuilderx.Reconciler = &dryRunner{} diff --git a/controllers/trace/dry_run_handler_test.go b/controllers/trace/dry_run_handler_test.go new file mode 100644 index 00000000000..120a3cdf33b --- /dev/null +++ b/controllers/trace/dry_run_handler_test.go @@ -0,0 +1,303 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + "encoding/json" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/golang/mock/gomock" + "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/constant" + "github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx" + testutil "github.com/apecloud/kubeblocks/pkg/testutil/k8s" + "github.com/apecloud/kubeblocks/pkg/testutil/k8s/mocks" +) + +var _ = Describe("dry_run_handler test", func() { + var ( + k8sMock *mocks.MockClient + controller *gomock.Controller + ) + + BeforeEach(func() { + controller, k8sMock = testutil.SetupK8sMock() + }) + + AfterEach(func() { + controller.Finish() + }) + + Context("Testing dry_run_handler", func() { + It("should work well", func() { + clusterDefinition := &kbappsv1.ClusterDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + Generation: int64(1), + }, + Spec: kbappsv1.ClusterDefinitionSpec{ + Topologies: []kbappsv1.ClusterTopology{ + { + Name: name, + Default: true, + Components: []kbappsv1.ClusterTopologyComponent{ + { + Name: name, + CompDef: name, + }, + }, + }, + }, + }, + Status: kbappsv1.ClusterDefinitionStatus{ + ObservedGeneration: int64(1), + Phase: kbappsv1.AvailablePhase, + }, + } + serviceVersion := "1.0.0" + componentDefinition := &kbappsv1.ComponentDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + Generation: int64(1), + }, + Spec: kbappsv1.ComponentDefinitionSpec{ + ServiceVersion: serviceVersion, + Runtime: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: name, + Image: "busybox", + }}, + }, + }, + Status: kbappsv1.ComponentDefinitionStatus{ + ObservedGeneration: int64(1), + Phase: kbappsv1.AvailablePhase, + }, + } + componentVersion := &kbappsv1.ComponentVersion{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + Generation: int64(1), + Annotations: map[string]string{ + "componentversion.kubeblocks.io/compatible-definitions": name, + }, + }, + Spec: kbappsv1.ComponentVersionSpec{ + CompatibilityRules: []kbappsv1.ComponentVersionCompatibilityRule{{ + CompDefs: []string{name}, + Releases: []string{name}, + }}, + Releases: []kbappsv1.ComponentVersionRelease{{ + Name: name, + ServiceVersion: serviceVersion, + Images: map[string]string{ + name: "busybox", + }, + }}, + }, + Status: kbappsv1.ComponentVersionStatus{ + ObservedGeneration: int64(1), + Phase: kbappsv1.AvailablePhase, + }, + } + clusterTemplate := &kbappsv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + UID: uid, + ResourceVersion: "1", + }, + Spec: kbappsv1.ClusterSpec{ + ClusterDef: name, + TerminationPolicy: kbappsv1.WipeOut, + ComponentSpecs: []kbappsv1.ClusterComponentSpec{{ + Name: name, + Replicas: 0, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + VolumeClaimTemplates: []kbappsv1.ClusterComponentVolumeClaimTemplate{{ + Name: name, + Spec: kbappsv1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("20Gi"), + }, + }, + }, + }}, + }}, + }, + } + primaryV1 := clusterTemplate.DeepCopy() + primaryV1.Generation = 1 + primaryV1.ResourceVersion = "2" + primaryV1.Status.Phase = kbappsv1.RunningClusterPhase + primaryV2 := clusterTemplate.DeepCopy() + primaryV2.Generation = 2 + primaryV2.ResourceVersion = "3" + primaryV2.Spec.ComponentSpecs[0].Replicas = 1 + specJSON, err := json.Marshal(primaryV2.Spec) + Expect(err).ToNot(HaveOccurred()) + var specMap map[string]interface{} + Expect(json.Unmarshal(specJSON, &specMap)).To(Succeed()) + desiredSpec, err := yaml.Marshal(specMap) + Expect(err).NotTo(HaveOccurred()) + + trace := &tracev1.ReconciliationTrace{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: tracev1.ReconciliationTraceSpec{ + TargetObject: &tracev1.ObjectReference{ + Namespace: primaryV1.Namespace, + Name: primaryV1.Name, + }, + DryRun: &tracev1.DryRun{ + DesiredSpec: string(desiredSpec), + }, + }, + } + + tree := kubebuilderx.NewObjectTree() + tree.SetRoot(trace) + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &kbappsv1.ClusterDefinition{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *kbappsv1.ClusterDefinition, _ ...client.GetOption) error { + *obj = *clusterDefinition.DeepCopy() + return nil + }).AnyTimes() + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &kbappsv1.ComponentDefinition{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *kbappsv1.ComponentDefinition, _ ...client.GetOption) error { + *obj = *componentDefinition.DeepCopy() + return nil + }).AnyTimes() + k8sMock.EXPECT(). + List(gomock.Any(), &kbappsv1.ComponentDefinitionList{}, gomock.Any()). + DoAndReturn(func(_ context.Context, list *kbappsv1.ComponentDefinitionList, _ ...client.GetOption) error { + list.Items = []kbappsv1.ComponentDefinition{*componentDefinition.DeepCopy()} + return nil + }).AnyTimes() + k8sMock.EXPECT(). + List(gomock.Any(), &kbappsv1.ComponentVersionList{}, gomock.Any()). + DoAndReturn(func(_ context.Context, list *kbappsv1.ComponentVersionList, _ ...client.ListOption) error { + list.Items = []kbappsv1.ComponentVersion{*componentVersion.DeepCopy()} + return nil + }).AnyTimes() + k8sMock.EXPECT(). + List(gomock.Any(), &kbappsv1.SidecarDefinitionList{}, gomock.Any()). + DoAndReturn(func(_ context.Context, list *kbappsv1.SidecarDefinitionList, _ ...client.ListOption) error { + return nil + }).AnyTimes() + k8sMock.EXPECT(). + List(gomock.Any(), &dpv1alpha1.BackupPolicyTemplateList{}, gomock.Any()). + DoAndReturn(func(_ context.Context, list *dpv1alpha1.BackupPolicyTemplateList, _ ...client.ListOption) error { + return nil + }).AnyTimes() + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &kbappsv1.Cluster{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *kbappsv1.Cluster, _ ...client.GetOption) error { + *obj = *primaryV1.DeepCopy() + return nil + }).Times(1) + k8sMock.EXPECT(). + List(gomock.Any(), &kbappsv1.ComponentList{}, gomock.Any()). + DoAndReturn(func(_ context.Context, list *kbappsv1.ComponentList, _ ...client.ListOption) error { + return nil + }).AnyTimes() + k8sMock.EXPECT(). + List(gomock.Any(), &corev1.ServiceList{}, gomock.Any()). + DoAndReturn(func(_ context.Context, list *corev1.ServiceList, _ ...client.ListOption) error { + return nil + }).AnyTimes() + k8sMock.EXPECT(). + List(gomock.Any(), &corev1.SecretList{}, gomock.Any()). + DoAndReturn(func(_ context.Context, list *corev1.SecretList, _ ...client.ListOption) error { + return nil + }).AnyTimes() + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &kbappsv1.Component{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *kbappsv1.Component, _ ...client.GetOption) error { + return apierrors.NewNotFound(kbappsv1.Resource(kbappsv1.ComponentKind), objKey.Name) + }).AnyTimes() + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &appsv1alpha1.Configuration{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *appsv1alpha1.Configuration, _ ...client.GetOption) error { + return apierrors.NewNotFound(appsv1alpha1.Resource(constant.ConfigurationKind), objKey.Name) + }).AnyTimes() + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &corev1.ConfigMap{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *corev1.ConfigMap, _ ...client.GetOption) error { + return apierrors.NewNotFound(corev1.Resource(constant.ConfigMapKind), objKey.Name) + }).AnyTimes() + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &corev1.PersistentVolumeClaim{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *corev1.PersistentVolumeClaim, _ ...client.GetOption) error { + return apierrors.NewNotFound(corev1.Resource(constant.PersistentVolumeClaimKind), objKey.Name) + }).AnyTimes() + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &corev1.PersistentVolume{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *corev1.PersistentVolume, _ ...client.GetOption) error { + return apierrors.NewNotFound(corev1.Resource(constant.PersistentVolumeKind), objKey.Name) + }).AnyTimes() + k8sMock.EXPECT().Scheme().Return(scheme.Scheme).AnyTimes() + + reconciler := dryRun(ctx, k8sMock, scheme.Scheme) + Expect(reconciler.PreCondition(tree)).To(Equal(kubebuilderx.ConditionSatisfied)) + res, err := reconciler.Reconcile(tree) + Expect(err).ToNot(HaveOccurred()) + Expect(res).Should(Equal(kubebuilderx.Continue)) + Expect(trace.Status.DryRunResult).ShouldNot(BeNil()) + Expect(trace.Status.DryRunResult.Phase).Should(Equal(tracev1.DryRunSucceedPhase)) + Expect(trace.Status.DryRunResult.DesiredSpecRevision).ShouldNot(BeEmpty()) + Expect(trace.Status.DryRunResult.ObservedTargetGeneration).Should(Equal(primaryV1.Generation)) + Expect(trace.Status.DryRunResult.SpecDiff).ShouldNot(BeEmpty()) + Expect(trace.Status.DryRunResult.Plan.ObjectTree).ShouldNot(BeNil()) + Expect(len(trace.Status.DryRunResult.Plan.Changes)).Should(BeNumerically(">", 0)) + Expect(trace.Status.DryRunResult.Plan.Summary.ObjectSummaries).ShouldNot(BeNil()) + }) + }) +}) diff --git a/controllers/trace/finalizer_handler.go b/controllers/trace/finalizer_handler.go new file mode 100644 index 00000000000..2d4324a1aae --- /dev/null +++ b/controllers/trace/finalizer_handler.go @@ -0,0 +1,52 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx" + "github.com/apecloud/kubeblocks/pkg/controller/model" +) + +type finalizerHandler struct{} + +func (f *finalizerHandler) PreCondition(tree *kubebuilderx.ObjectTree) *kubebuilderx.CheckResult { + if tree.GetRoot() == nil || model.IsObjectDeleting(tree.GetRoot()) { + return kubebuilderx.ConditionUnsatisfied + } + for _, f := range tree.GetRoot().GetFinalizers() { + if f == finalizer { + return kubebuilderx.ConditionUnsatisfied + } + } + return kubebuilderx.ConditionSatisfied +} + +func (f *finalizerHandler) Reconcile(tree *kubebuilderx.ObjectTree) (kubebuilderx.Result, error) { + finalizers := tree.GetRoot().GetFinalizers() + finalizers = append(finalizers, finalizer) + tree.GetRoot().SetFinalizers(finalizers) + return kubebuilderx.Commit, nil +} + +func assureFinalizer() kubebuilderx.Reconciler { + return &finalizerHandler{} +} + +var _ kubebuilderx.Reconciler = &finalizerHandler{} diff --git a/controllers/trace/finalizer_handler_test.go b/controllers/trace/finalizer_handler_test.go new file mode 100644 index 00000000000..99f8d9683ca --- /dev/null +++ b/controllers/trace/finalizer_handler_test.go @@ -0,0 +1,56 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx" +) + +var _ = Describe("finalizer_handler test", func() { + Context("Testing finalizer_handler", func() { + It("should work well", func() { + trace := &tracev1.ReconciliationTrace{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + Generation: int64(1), + }, + } + + tree := kubebuilderx.NewObjectTree() + tree.SetRoot(trace) + + reconciler := assureFinalizer() + Expect(reconciler.PreCondition(tree).Err).ToNot(HaveOccurred()) + Expect(reconciler.PreCondition(tree).Satisfied).To(BeTrue()) + res, err := reconciler.Reconcile(tree) + Expect(err).ToNot(HaveOccurred()) + Expect(res).To(Equal(kubebuilderx.Commit)) + Expect(tree.GetRoot().GetFinalizers()).To(HaveLen(1)) + Expect(tree.GetRoot().GetFinalizers()[0]).To(Equal(finalizer)) + }) + }) +}) diff --git a/controllers/trace/i18n_resources_manager.go b/controllers/trace/i18n_resources_manager.go new file mode 100644 index 00000000000..2d125b4d101 --- /dev/null +++ b/controllers/trace/i18n_resources_manager.go @@ -0,0 +1,111 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "fmt" + "strings" + "sync" + + "golang.org/x/text/language" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type I18nResourcesManager interface { + ParseRaw(*corev1.ConfigMap) error + GetFormatString(key string, locale string) string +} + +type i18nResourcesManager struct { + resources map[string]map[string]string + resourceMapLock sync.RWMutex + + parsedRawSet sets.Set[client.ObjectKey] + rawSetLock sync.Mutex +} + +var defaultResourcesManager = &i18nResourcesManager{ + resources: make(map[string]map[string]string), + parsedRawSet: sets.New[client.ObjectKey](), +} + +func (m *i18nResourcesManager) ParseRaw(cm *corev1.ConfigMap) error { + if cm == nil { + return nil + } + + m.rawSetLock.Lock() + defer m.rawSetLock.Unlock() + + if m.parsedRawSet.Has(client.ObjectKeyFromObject(cm)) { + return nil + } + + m.resourceMapLock.Lock() + defer m.resourceMapLock.Unlock() + + for key, value := range cm.Data { + _, err := language.Parse(key) + if err != nil { + return err + } + locale := strings.ToLower(key) + resourceLocaleMap, ok := m.resources[locale] + if !ok { + resourceLocaleMap = make(map[string]string) + } + formatedStrings := strings.Split(value, "\n") + for _, formatedString := range formatedStrings { + if len(formatedString) == 0 { + continue + } + index := strings.Index(formatedString, "=") + if index <= 0 { + return fmt.Errorf("can't parse string %s as a key=value pair", formatedString) + } + resourceKey := formatedString[:index] + resourceValue := formatedString[index+1:] + if len(resourceValue) == 0 { + return fmt.Errorf("can't parse string %s as a key=value pair", formatedString) + } + resourceLocaleMap[resourceKey] = resourceValue + } + m.resources[locale] = resourceLocaleMap + } + + m.parsedRawSet.Insert(client.ObjectKeyFromObject(cm)) + + return nil +} + +func (m *i18nResourcesManager) GetFormatString(key string, locale string) string { + m.resourceMapLock.RLock() + defer m.resourceMapLock.RUnlock() + + resourceLocaleMap, ok := m.resources[strings.ToLower(locale)] + if !ok { + return "" + } + return resourceLocaleMap[key] +} + +var _ I18nResourcesManager = &i18nResourcesManager{} diff --git a/controllers/trace/informer_manager.go b/controllers/trace/informer_manager.go new file mode 100644 index 00000000000..fe53a308634 --- /dev/null +++ b/controllers/trace/informer_manager.go @@ -0,0 +1,230 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + "fmt" + "sync" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/workqueue" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" + + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/constant" +) + +type InformerManager interface { + Start(context.Context) error +} + +type informerManager struct { + once sync.Once + + eventChan chan event.GenericEvent + + informerSet sets.Set[schema.GroupVersionKind] + informerSetLock sync.Mutex + + cache cache.Cache + cli client.Client + ctx context.Context + + handler handler.EventHandler + + // Queue is an listeningQueue that listens for events from Informers and adds object keys to + // the Queue for processing + queue workqueue.RateLimitingInterface + + scheme *runtime.Scheme + + logger logr.Logger +} + +func (m *informerManager) Start(ctx context.Context) error { + m.ctx = ctx + + if err := m.watchKubeBlocksRelatedResources(); err != nil { + return err + } + + m.once.Do(func() { + go func() { + for m.processNextWorkItem() { + } + }() + }) + + return nil +} + +func (m *informerManager) watch(resource schema.GroupVersionKind) error { + m.informerSetLock.Lock() + defer m.informerSetLock.Unlock() + + if _, ok := m.informerSet[resource]; ok { + return nil + } + if err := m.createInformer(resource); err != nil { + return err + } + m.informerSet.Insert(resource) + + return nil +} + +// processNextWorkItem will read a single work item off the workqueue and +// attempt to process it, by calling the reconcileHandler. +func (m *informerManager) processNextWorkItem() bool { + obj, shutdown := m.queue.Get() + if shutdown { + // Stop working + m.logger.Error(fmt.Errorf("informer queue is shutdown"), "") + return false + } + + defer m.queue.Done(obj) + + var object client.Object + switch o := obj.(type) { + case event.CreateEvent: + object = o.Object + case event.UpdateEvent: + object = o.ObjectNew + case event.DeleteEvent: + object = o.Object + case event.GenericEvent: + object = o.Object + } + // get involved object if 'object' is an Event + if evt, ok := object.(*corev1.Event); ok { + ro, err := m.scheme.New(evt.InvolvedObject.GroupVersionKind()) + if err != nil { + m.logger.Error(err, "new an event involved object failed") + return true + } + object, _ = ro.(client.Object) + err = m.cli.Get(context.Background(), client.ObjectKey{Namespace: evt.InvolvedObject.Namespace, Name: evt.InvolvedObject.Name}, object) + if err != nil && !apierrors.IsNotFound(err) { + m.logger.Error(err, "get involved object failed: %s", evt.InvolvedObject) + return true + } + } + if object != nil { + m.eventChan <- event.GenericEvent{Object: object} + } + + return true +} + +func (m *informerManager) createInformer(gvk schema.GroupVersionKind) error { + o, err := m.scheme.New(gvk) + if err != nil { + return err + } + obj, ok := o.(client.Object) + if !ok { + return fmt.Errorf("can't find object of type %s", gvk) + } + src := source.Kind(m.cache, obj) + return src.Start(m.ctx, m.handler, m.queue) +} + +type eventProxy struct{} + +func (e *eventProxy) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) { + q.Add(evt) +} + +func (e *eventProxy) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) { + q.Add(evt) +} + +func (e *eventProxy) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { + q.Add(evt) +} + +func (e *eventProxy) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) { + q.Add(evt) +} + +func NewInformerManager(cli client.Client, cache cache.Cache, scheme *runtime.Scheme, eventChan chan event.GenericEvent) InformerManager { + return &informerManager{ + cli: cli, + cache: cache, + scheme: scheme, + eventChan: eventChan, + handler: &eventProxy{}, + informerSet: sets.New[schema.GroupVersionKind](), + queue: workqueue.NewRateLimitingQueueWithConfig(workqueue.DefaultControllerRateLimiter(), workqueue.RateLimitingQueueConfig{ + Name: "informer-manager", + }), + logger: ctrl.Log.WithName("informer-manager"), + } +} + +func (m *informerManager) watchKubeBlocksRelatedResources() error { + gvks := sets.New[schema.GroupVersionKind]() + parseGVK := func(ot *tracev1.ObjectType) error { + gvk, err := objectTypeToGVK(ot) + if err != nil { + return err + } + gvks.Insert(*gvk) + return nil + } + // watch corev1.Event + if err := parseGVK(&tracev1.ObjectType{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: constant.EventKind, + }); err != nil { + return err + } + for _, rule := range getKBOwnershipRules() { + if err := parseGVK(&rule.Primary); err != nil { + return err + } + for _, resource := range rule.OwnedResources { + if err := parseGVK(&resource.Secondary); err != nil { + return err + } + } + } + for gvk := range gvks { + if err := m.watch(gvk); err != nil { + return err + } + } + return nil +} + +var _ InformerManager = &informerManager{} +var _ handler.EventHandler = &eventProxy{} diff --git a/controllers/trace/mock_client.go b/controllers/trace/mock_client.go new file mode 100644 index 00000000000..1e2a100f2fc --- /dev/null +++ b/controllers/trace/mock_client.go @@ -0,0 +1,389 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strings" + + jsonpatch "github.com/evanphx/json-patch/v5" + "github.com/google/go-cmp/cmp" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/strategicpatch" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" +) + +type mockClient struct { + realClient client.Client + subResourceClient client.SubResourceWriter + store ChangeCaptureStore + managedGVK sets.Set[schema.GroupVersionKind] +} + +type mockSubResourceClient struct { + store ChangeCaptureStore + scheme *runtime.Scheme +} + +func (c *mockClient) Get(ctx context.Context, objKey client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + objectRef, err := getObjectRef(obj, c.realClient.Scheme()) + if err != nil { + return err + } + objectRef.ObjectKey = objKey + res := c.store.Get(objectRef) + if res == nil { + return c.realClient.Get(ctx, objKey, obj, opts...) + } + return copyObj(res, obj) +} + +func (c *mockClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + // Get the GVK for the list type + gvk, err := apiutil.GVKForObject(list, c.realClient.Scheme()) + if err != nil { + return fmt.Errorf("failed to get GVK for list: %w", err) + } + gvk.Kind, _ = strings.CutSuffix(gvk.Kind, "List") + + if !c.managedGVK.Has(gvk) { + return c.realClient.List(ctx, list, opts...) + } + + // Get the objects of the same GVK from the store + objects := c.store.List(&gvk) + + // Iterate over stored objects and add them to the list + items, err := meta.ExtractList(list) + if err != nil { + return fmt.Errorf("failed to extract list: %w", err) + } + + // Extract namespace and other options from opts + listOptions := &client.ListOptions{} + listOptions.ApplyOptions(opts) + + for _, obj := range objects { + if listOptions.Namespace != "" && obj.GetNamespace() != listOptions.Namespace { + continue + } + if listOptions.LabelSelector != nil && !listOptions.LabelSelector.Matches(labels.Set(obj.GetLabels())) { + continue + } + items = append(items, obj.DeepCopyObject()) + } + + // Set the items back to the list + if err := meta.SetList(list, items); err != nil { + return fmt.Errorf("failed to set list: %w", err) + } + + return nil +} + +func (c *mockClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + objectRef, err := getObjectRef(obj, c.realClient.Scheme()) + if err != nil { + return err + } + o := c.store.Get(objectRef) + if o != nil { + return apierrors.NewAlreadyExists(objectRef.GroupVersion().WithResource(objectRef.Kind).GroupResource(), fmt.Sprintf("%s/%s", objectRef.Namespace, objectRef.Name)) + } + obj.SetGeneration(1) + return c.store.Insert(obj) +} + +func (c *mockClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + objectRef, err := getObjectRef(obj, c.realClient.Scheme()) + if err != nil { + return err + } + object := c.store.Get(objectRef) + if object == nil { + return nil + } + if object.GetDeletionTimestamp() == nil { + ts := metav1.Now() + object.SetDeletionTimestamp(&ts) + return c.store.Update(object) + } + if len(object.GetFinalizers()) != 0 { + return nil + } + return c.store.Delete(obj) +} + +func (c *mockClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + objectRef, err := getObjectRef(obj, c.realClient.Scheme()) + if err != nil { + return err + } + oldObj := c.store.Get(objectRef) + if oldObj == nil { + return apierrors.NewNotFound(objectRef.GroupVersion().WithResource(objectRef.Kind).GroupResource(), fmt.Sprintf("%s/%s", objectRef.Namespace, objectRef.Name)) + } + metaChanged := checkMetadata(oldObj, obj) + specChanged := increaseGeneration(oldObj, obj) + if metaChanged || specChanged { + return c.store.Update(obj) + } + return nil +} + +func (c *mockClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + return doPatch(obj, patch, c.store, c.realClient.Scheme(), true) +} + +func doPatch(obj client.Object, patch client.Patch, store ChangeCaptureStore, scheme *runtime.Scheme, checkGeneration bool) error { + objectRef, err := getObjectRef(obj, scheme) + if err != nil { + return err + } + oldObj := store.Get(objectRef) + if oldObj == nil { + return apierrors.NewNotFound(objectRef.GroupVersion().WithResource(objectRef.Kind).GroupResource(), fmt.Sprintf("%s/%s", objectRef.Namespace, objectRef.Name)) + } + patchData, err := patch.Data(obj) + if err != nil { + return err + } + newObj, err := applyPatch(oldObj, patch.Type(), patchData) + if err != nil { + return err + } + metaChanged := checkMetadata(oldObj, newObj) + specChanged := false + statusChanged := false + if checkGeneration { + specChanged = increaseGeneration(oldObj, newObj) + } else { + statusChanged = checkStatus(oldObj, newObj) + } + if metaChanged || specChanged || statusChanged { + return store.Update(newObj) + } + return nil +} + +func applyPatch(obj client.Object, patchType types.PatchType, patchData []byte) (client.Object, error) { + // Convert the object to JSON + originalJSON, err := json.Marshal(obj) + if err != nil { + return nil, fmt.Errorf("failed to marshal original object: %w", err) + } + + // Apply the patch + var patchedJSON []byte + switch patchType { + case types.StrategicMergePatchType: + patchedJSON, err = strategicpatch.StrategicMergePatch(originalJSON, patchData, obj) + case types.MergePatchType: + patchedJSON, err = jsonpatch.MergePatch(originalJSON, patchData) + default: + return nil, fmt.Errorf("unsupported patch type: %s", patchType) + } + if err != nil { + return nil, fmt.Errorf("failed to apply patch: %w", err) + } + + // Unmarshal the patched JSON into a new clean object + newObj, err := newZeroObject(obj) + if err != nil { + return nil, err + } + err = json.Unmarshal(patchedJSON, newObj) + return newObj, err +} + +// newZeroObject creates a new zeroed instance of the same type as the given object +func newZeroObject(obj client.Object) (client.Object, error) { + if obj == nil { + return nil, fmt.Errorf("input object cannot be nil") + } + + // Get the reflect.Type of the original object + t := reflect.TypeOf(obj) + if t.Kind() != reflect.Ptr { + return nil, fmt.Errorf("object must be a pointer") + } + + // Get the underlying type (struct type) + elemType := t.Elem() + if elemType.Kind() != reflect.Struct { + return nil, fmt.Errorf("object must be a pointer to struct") + } + + // Create new instance + newObj := reflect.New(elemType).Interface() + + // Type assert to client.Object + clientObj, ok := newObj.(client.Object) + if !ok { + return nil, fmt.Errorf("failed to convert new instance to client.Object") + } + + return clientObj, nil +} + +func checkMetadata(oldObj client.Object, newObj client.Object) bool { + if !reflect.DeepEqual(oldObj.GetFinalizers(), newObj.GetFinalizers()) || + !reflect.DeepEqual(oldObj.GetOwnerReferences(), newObj.GetOwnerReferences()) || + !reflect.DeepEqual(oldObj.GetAnnotations(), newObj.GetAnnotations()) || + !reflect.DeepEqual(oldObj.GetLabels(), newObj.GetLabels()) { + return true + } + return false +} + +func increaseGeneration(oldObj, newObj client.Object) bool { + oldObjCopy, _ := normalize(oldObj) + newObjCopy, _ := normalize(newObj) + if oldObjCopy == nil || newObjCopy == nil { + return false + } + oldSpec, _ := getFieldAsStruct(oldObjCopy, specFieldName) + newSpec, _ := getFieldAsStruct(newObjCopy, specFieldName) + if oldSpec == nil || newSpec == nil { + return false + } + diff := cmp.Diff(oldSpec, newSpec) + if diff == "" { + return false + } + newObj.SetGeneration(newObj.GetGeneration() + 1) + return true +} + +func checkStatus(oldObj client.Object, newObj client.Object) bool { + oldStatus, _ := getFieldAsStruct(oldObj, statusFieldName) + newStatus, _ := getFieldAsStruct(newObj, statusFieldName) + if oldStatus == nil || newStatus == nil { + return false + } + return !reflect.DeepEqual(oldStatus, newStatus) +} + +func (c *mockClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + return fmt.Errorf("DeleteAllOf not implemented") +} + +func (c *mockClient) Status() client.SubResourceWriter { + return c.subResourceClient +} + +func (c *mockClient) SubResource(subResource string) client.SubResourceClient { + panic("not implemented") +} + +func (c *mockClient) Scheme() *runtime.Scheme { + return c.realClient.Scheme() +} + +func (c *mockClient) RESTMapper() meta.RESTMapper { + return c.realClient.RESTMapper() +} + +func (c *mockClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return c.realClient.GroupVersionKindFor(obj) +} + +func (c *mockClient) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return c.realClient.IsObjectNamespaced(obj) +} + +func (c *mockSubResourceClient) Create(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error { + panic("not implemented") +} + +func (c *mockSubResourceClient) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + objectRef, err := getObjectRef(obj, c.scheme) + if err != nil { + return err + } + oldObj := c.store.Get(objectRef) + if oldObj == nil { + return apierrors.NewNotFound(objectRef.GroupVersion().WithResource(objectRef.Kind).GroupResource(), fmt.Sprintf("%s/%s", objectRef.Namespace, objectRef.Name)) + } + if checkStatus(oldObj, obj) { + return c.store.Update(obj) + } + return nil +} + +func (c *mockSubResourceClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + return doPatch(obj, patch, c.store, c.scheme, false) +} + +func newMockClient(realClient client.Client, store ChangeCaptureStore, rules []OwnershipRule) (client.Client, error) { + managedGVK := sets.New[schema.GroupVersionKind]() + addToManaged := func(objType *tracev1.ObjectType) error { + gvk, err := objectTypeToGVK(objType) + if err != nil { + return err + } + managedGVK.Insert(*gvk) + return nil + } + for _, rule := range rules { + if err := addToManaged(&rule.Primary); err != nil { + return nil, err + } + for _, ownedResource := range rule.OwnedResources { + if err := addToManaged(&ownedResource.Secondary); err != nil { + return nil, err + } + } + } + + return &mockClient{ + realClient: realClient, + store: store, + managedGVK: managedGVK, + subResourceClient: &mockSubResourceClient{ + store: store, + scheme: realClient.Scheme(), + }, + }, nil +} + +func copyObj(src, dst client.Object) error { + srcJSON, err := json.Marshal(src) + if err != nil { + return fmt.Errorf("failed to marshal src object: %w", err) + } + return json.Unmarshal(srcJSON, dst) +} + +var _ client.Client = &mockClient{} +var _ client.SubResourceWriter = &mockSubResourceClient{} diff --git a/controllers/trace/mock_event_recorder.go b/controllers/trace/mock_event_recorder.go new file mode 100644 index 00000000000..0f150b7ae12 --- /dev/null +++ b/controllers/trace/mock_event_recorder.go @@ -0,0 +1,89 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +type mockEventRecorder struct { + store ChangeCaptureStore + logger logr.Logger +} + +func (r *mockEventRecorder) Event(object runtime.Object, eventtype, reason, message string) { + r.emitEvent(object, nil, eventtype, reason, message) +} + +func (r *mockEventRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { + message := fmt.Sprintf(messageFmt, args...) + r.emitEvent(object, nil, eventtype, reason, message) +} + +func (r *mockEventRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) { + message := fmt.Sprintf(messageFmt, args...) + r.emitEvent(object, annotations, eventtype, reason, message) +} + +func (r *mockEventRecorder) emitEvent(object runtime.Object, annotations map[string]string, eventtype, reason, message string) { + metaObj, err := meta.Accessor(object) + if err != nil { + r.logger.Error(err, "Error accessing object metadata") + return + } + + event := &corev1.Event{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s.%s", metaObj.GetName(), string(uuid.NewUUID())), + Namespace: metaObj.GetNamespace(), + Annotations: annotations, + }, + InvolvedObject: corev1.ObjectReference{ + Kind: object.GetObjectKind().GroupVersionKind().Kind, + Namespace: metaObj.GetNamespace(), + Name: metaObj.GetName(), + UID: metaObj.GetUID(), + }, + Type: eventtype, + Reason: reason, + Message: message, + } + _ = r.store.Insert(event) +} + +func newMockEventRecorder(store ChangeCaptureStore) record.EventRecorder { + logger := log.FromContext(context.Background()).WithName("MockEventRecorder") + return &mockEventRecorder{ + store: store, + logger: logger, + } +} + +var _ record.EventRecorder = &mockEventRecorder{} diff --git a/controllers/trace/object_revision_store.go b/controllers/trace/object_revision_store.go new file mode 100644 index 00000000000..7c5dbf03b7e --- /dev/null +++ b/controllers/trace/object_revision_store.go @@ -0,0 +1,187 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "strings" + "sync" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/apecloud/kubeblocks/pkg/controller/model" +) + +// ObjectRevisionStore defines an object store which can get the history revision. +// WARN: This store is designed only for Reconciliation Trace Controller, +// it's not thread-safe, it doesn't do a deep copy before returning the object. +// Don't use it in other place. +type ObjectRevisionStore interface { + Insert(object, reference client.Object) error + Get(objectRef *model.GVKNObjKey, revision int64) (client.Object, error) + List(gvk *schema.GroupVersionKind) map[types.NamespacedName]map[int64]client.Object + Delete(objectRef *model.GVKNObjKey, reference client.Object, revision int64) +} + +type objectRevisionStore struct { + store map[schema.GroupVersionKind]map[types.NamespacedName]map[int64]client.Object + storeLock sync.RWMutex + + referenceCounter map[revisionObjectRef]sets.Set[types.UID] + counterLock sync.Mutex + + scheme *runtime.Scheme +} + +type revisionObjectRef struct { + model.GVKNObjKey + revision int64 +} + +func (s *objectRevisionStore) Insert(object, reference client.Object) error { + // insert into store + s.storeLock.Lock() + defer s.storeLock.Unlock() + + objectRef, err := getObjectRef(object, s.scheme) + if err != nil { + return err + } + objectMap, ok := s.store[objectRef.GroupVersionKind] + if !ok { + objectMap = make(map[types.NamespacedName]map[int64]client.Object) + s.store[objectRef.GroupVersionKind] = objectMap + } + revisionMap, ok := objectMap[objectRef.ObjectKey] + if !ok { + revisionMap = make(map[int64]client.Object) + objectMap[objectRef.ObjectKey] = revisionMap + } + revision := parseRevision(object.GetResourceVersion()) + revisionMap[revision] = object + + // update reference counter + s.counterLock.Lock() + defer s.counterLock.Unlock() + + revObjectRef := revisionObjectRef{ + GVKNObjKey: *objectRef, + revision: revision, + } + referenceMap, ok := s.referenceCounter[revObjectRef] + if !ok { + referenceMap = sets.New[types.UID]() + } + referenceMap.Insert(reference.GetUID()) + s.referenceCounter[revObjectRef] = referenceMap + + return nil +} + +func (s *objectRevisionStore) Get(objectRef *model.GVKNObjKey, revision int64) (client.Object, error) { + s.storeLock.RLock() + defer s.storeLock.RUnlock() + + objectMap, ok := s.store[objectRef.GroupVersionKind] + if !ok { + return nil, apierrors.NewNotFound(objectRef.GroupVersion().WithResource(strings.ToLower(objectRef.Kind)).GroupResource(), objectRef.Name) + } + revisionMap, ok := objectMap[objectRef.ObjectKey] + if !ok { + return nil, apierrors.NewNotFound(objectRef.GroupVersion().WithResource(strings.ToLower(objectRef.Kind)).GroupResource(), objectRef.Name) + } + object, ok := revisionMap[revision] + if !ok { + return nil, apierrors.NewNotFound(objectRef.GroupVersion().WithResource(strings.ToLower(objectRef.Kind)).GroupResource(), objectRef.Name) + } + object = object.DeepCopyObject().(client.Object) + return object, nil +} + +func (s *objectRevisionStore) List(gvk *schema.GroupVersionKind) map[types.NamespacedName]map[int64]client.Object { + s.storeLock.RLock() + defer s.storeLock.RUnlock() + + objectMap, ok := s.store[*gvk] + if !ok { + return nil + } + objectMapCopy := make(map[types.NamespacedName]map[int64]client.Object, len(objectMap)) + for name, revisionMap := range objectMap { + revisionMapCopy := make(map[int64]client.Object, len(revisionMap)) + objectMapCopy[name] = revisionMapCopy + for revision, object := range revisionMap { + objectCopy := object.DeepCopyObject().(client.Object) + revisionMapCopy[revision] = objectCopy + } + } + return objectMapCopy +} + +func (s *objectRevisionStore) Delete(objectRef *model.GVKNObjKey, reference client.Object, revision int64) { + s.storeLock.Lock() + defer s.storeLock.Unlock() + s.counterLock.Lock() + defer s.counterLock.Unlock() + + // decrease reference counter + revObjectRef := revisionObjectRef{ + GVKNObjKey: *objectRef, + revision: revision, + } + referenceMap, ok := s.referenceCounter[revObjectRef] + if ok { + referenceMap.Delete(reference.GetUID()) + } + if len(referenceMap) > 0 { + return + } + + // delete object + objectMap, ok := s.store[objectRef.GroupVersionKind] + if !ok { + return + } + revisionMap, ok := objectMap[objectRef.ObjectKey] + if !ok { + return + } + delete(revisionMap, revision) + if len(referenceMap) == 0 { + delete(objectMap, objectRef.ObjectKey) + } + if len(objectMap) == 0 { + delete(s.store, objectRef.GroupVersionKind) + } +} + +func NewObjectStore(scheme *runtime.Scheme) ObjectRevisionStore { + return &objectRevisionStore{ + store: make(map[schema.GroupVersionKind]map[types.NamespacedName]map[int64]client.Object), + referenceCounter: make(map[revisionObjectRef]sets.Set[types.UID]), + scheme: scheme, + } +} + +var _ ObjectRevisionStore = &objectRevisionStore{} diff --git a/controllers/trace/object_revision_store_test.go b/controllers/trace/object_revision_store_test.go new file mode 100644 index 00000000000..4b591e67771 --- /dev/null +++ b/controllers/trace/object_revision_store_test.go @@ -0,0 +1,81 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/kubernetes/scheme" + + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" + "github.com/apecloud/kubeblocks/pkg/controller/builder" +) + +var _ = Describe("object_revision_store test", func() { + Context("Testing object_revision_store", func() { + It("should work well", func() { + store := NewObjectStore(scheme.Scheme) + + By("Insert a component") + primary := builder.NewClusterBuilder(namespace, name).SetUID(uid).SetResourceVersion(resourceVersion).GetObject() + compName := "test" + fullCompName := fmt.Sprintf("%s-%s", primary.Name, compName) + secondary := builder.NewComponentBuilder(namespace, fullCompName, ""). + SetOwnerReferences(kbappsv1.APIVersion, kbappsv1.ClusterKind, primary). + SetUID(uid). + GetObject() + secondary.ResourceVersion = resourceVersion + Expect(store.Insert(secondary, primary)).Should(Succeed()) + objectRef, err := getObjectRef(secondary, scheme.Scheme) + Expect(err).Should(BeNil()) + + By("Get the component with right revision") + revision := parseRevision(secondary.ResourceVersion) + obj, err := store.Get(objectRef, revision) + Expect(err).Should(BeNil()) + Expect(obj).Should(Equal(secondary)) + + By("Get the component with wrong revision") + _, err = store.Get(objectRef, revision+1) + Expect(err).ShouldNot(BeNil()) + Expect(apierrors.IsNotFound(err)).Should(BeTrue()) + + By("List all components") + objects := store.List(&objectRef.GroupVersionKind) + Expect(objects).Should(HaveLen(1)) + revisionMap, ok := objects[objectRef.ObjectKey] + Expect(ok).Should(BeTrue()) + Expect(revisionMap).Should(HaveLen(1)) + obj, ok = revisionMap[revision] + Expect(ok).Should(BeTrue()) + Expect(obj).Should(Equal(secondary)) + + By("Delete the component") + store.Delete(objectRef, primary, revision) + _, err = store.Get(objectRef, revision) + Expect(err).ShouldNot(BeNil()) + Expect(apierrors.IsNotFound(err)).Should(BeTrue()) + }) + }) +}) diff --git a/controllers/trace/object_tree_root_finder.go b/controllers/trace/object_tree_root_finder.go new file mode 100644 index 00000000000..f975b76b390 --- /dev/null +++ b/controllers/trace/object_tree_root_finder.go @@ -0,0 +1,166 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "container/list" + "context" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" +) + +type ObjectTreeRootFinder interface { + GetEventChannel() chan event.GenericEvent + GetEventHandler() handler.EventHandler +} + +type rootFinder struct { + client.Client + logger logr.Logger + eventChan chan event.GenericEvent +} + +func (f *rootFinder) GetEventChannel() chan event.GenericEvent { + return f.eventChan +} + +func (f *rootFinder) GetEventHandler() handler.EventHandler { + return handler.EnqueueRequestsFromMapFunc(f.findRoots) +} + +// findRoots finds the root(s) object of the 'object' by the object tree. +// The basic idea is, find the parent(s) of the object based on ownership rules defined in trace definition, +// and do this recursively until find all the root object(s). +func (f *rootFinder) findRoots(ctx context.Context, object client.Object) []reconcile.Request { + waitingList := list.New() + waitingList.PushFront(object) + + var roots []client.Object + primaryTypeList := []tracev1.ObjectType{rootObjectType} + for waitingList.Len() > 0 { + e := waitingList.Front() + waitingList.Remove(e) + obj, _ := e.Value.(client.Object) + objGVK, err := apiutil.GVKForObject(obj, f.Scheme()) + if err != nil { + f.logger.Error(err, "get GVK of %s/%s failed", obj.GetNamespace(), obj.GetName()) + return nil + } + found := false + for _, primaryType := range primaryTypeList { + gvk, err := objectTypeToGVK(&primaryType) + if err != nil { + f.logger.Error(err, "convert objectType %s to GVK failed", primaryType) + return nil + } + if objGVK == *gvk { + roots = append(roots, obj) + found = true + } + } + if found { + continue + } + rules := getKBOwnershipRules() + for i := range rules { + rule := &rules[i] + for _, resource := range rule.OwnedResources { + gvk, err := objectTypeToGVK(&resource.Secondary) + if err != nil { + f.logger.Error(err, "convert objectType %s to GVK failed", resource.Secondary) + return nil + } + if objGVK != *gvk { + continue + } + primaryGVK, err := objectTypeToGVK(&rule.Primary) + if err != nil { + f.logger.Error(err, "convert objectType %s to GVK failed", rule.Primary) + return nil + } + objectList, err := getObjectsByGVK(ctx, f, primaryGVK, nil) + if err != nil { + f.logger.Error(err, "getObjectsByGVK for GVK %s failed", primaryGVK) + return nil + } + for _, owner := range objectList { + opts, err := parseQueryOptions(owner, &resource.Criteria) + if err != nil { + f.logger.Error(err, "parse query options failed: %s", resource.Criteria) + return nil + } + if opts.match(obj) { + waitingList.PushBack(owner) + } + } + } + } + } + + clusterKeys := sets.New[client.ObjectKey]() + for _, root := range roots { + clusterKeys.Insert(client.ObjectKeyFromObject(root)) + } + + // list all trace objects, filter by result Cluster objects. + traceList := &tracev1.ReconciliationTraceList{} + if err := f.List(ctx, traceList); err != nil { + f.logger.Error(err, "list trace failed", "") + return nil + } + getTargetObjectKey := func(trace *tracev1.ReconciliationTrace) client.ObjectKey { + key := client.ObjectKeyFromObject(trace) + if trace.Spec.TargetObject != nil { + key.Namespace = trace.Spec.TargetObject.Namespace + key.Name = trace.Spec.TargetObject.Name + } + return key + } + var requests []reconcile.Request + for i := range traceList.Items { + trace := &traceList.Items[i] + key := getTargetObjectKey(trace) + if clusterKeys.Has(key) { + requests = append(requests, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(trace)}) + } + } + + return requests +} + +func NewObjectTreeRootFinder(cli client.Client) ObjectTreeRootFinder { + logger := log.FromContext(context.Background()).WithName("ObjectTreeRootFinder") + return &rootFinder{ + Client: cli, + logger: logger, + eventChan: make(chan event.GenericEvent), + } +} + +var _ ObjectTreeRootFinder = &rootFinder{} diff --git a/controllers/trace/object_tree_root_finder_test.go b/controllers/trace/object_tree_root_finder_test.go new file mode 100644 index 00000000000..a9e8a7a4765 --- /dev/null +++ b/controllers/trace/object_tree_root_finder_test.go @@ -0,0 +1,95 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/golang/mock/gomock" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/constant" + "github.com/apecloud/kubeblocks/pkg/controller/builder" + testutil "github.com/apecloud/kubeblocks/pkg/testutil/k8s" + "github.com/apecloud/kubeblocks/pkg/testutil/k8s/mocks" +) + +var _ = Describe("object_tree_root_finder test", func() { + var ( + k8sMock *mocks.MockClient + controller *gomock.Controller + ) + + BeforeEach(func() { + controller, k8sMock = testutil.SetupK8sMock() + }) + + AfterEach(func() { + controller.Finish() + }) + + Context("Testing object_tree_root_finder", func() { + It("should work well", func() { + trace := &tracev1.ReconciliationTrace{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + } + root := builder.NewClusterBuilder(namespace, name).SetUID(uid).SetResourceVersion(resourceVersion).GetObject() + compName := "test" + fullCompName := fmt.Sprintf("%s-%s", root.Name, compName) + secondary := builder.NewComponentBuilder(namespace, fullCompName, ""). + SetOwnerReferences(kbappsv1.APIVersion, kbappsv1.ClusterKind, root). + SetUID(uid). + AddLabels(constant.AppManagedByLabelKey, constant.AppName). + AddLabels(constant.AppInstanceLabelKey, root.Name). + GetObject() + + k8sMock.EXPECT(). + List(gomock.Any(), &kbappsv1.ClusterList{}, gomock.Any()). + DoAndReturn(func(_ context.Context, list *kbappsv1.ClusterList, opts ...client.ListOption) error { + list.Items = []kbappsv1.Cluster{*root} + return nil + }).Times(1) + k8sMock.EXPECT(). + List(gomock.Any(), &tracev1.ReconciliationTraceList{}, gomock.Any()). + DoAndReturn(func(_ context.Context, list *tracev1.ReconciliationTraceList, opts ...client.ListOption) error { + list.Items = []tracev1.ReconciliationTrace{*trace} + return nil + }).Times(1) + k8sMock.EXPECT().Scheme().Return(scheme.Scheme).AnyTimes() + + finder := NewObjectTreeRootFinder(k8sMock).(*rootFinder) + res := finder.findRoots(ctx, secondary) + Expect(res).Should(HaveLen(1)) + Expect(res[0]).Should(Equal(reconcile.Request{NamespacedName: client.ObjectKeyFromObject(root)})) + }) + }) +}) diff --git a/controllers/trace/plan_generator.go b/controllers/trace/plan_generator.go new file mode 100644 index 00000000000..d59844da977 --- /dev/null +++ b/controllers/trace/plan_generator.go @@ -0,0 +1,193 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + "fmt" + "time" + + "github.com/google/go-cmp/cmp" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/controller/model" +) + +type PlanGenerator interface { + generatePlan(desiredRoot *kbappsv1.Cluster) (*tracev1.DryRunResult, error) +} + +type objectLoader func() (map[model.GVKNObjKey]client.Object, error) +type descriptionFormatter func(client.Object, client.Object, tracev1.ObjectChangeType, *schema.GroupVersionKind) (string, *string) + +type planGenerator struct { + ctx context.Context + cli client.Client + scheme *runtime.Scheme + loader objectLoader + formatter descriptionFormatter +} + +func (g *planGenerator) generatePlan(desiredRoot *kbappsv1.Cluster) (*tracev1.DryRunResult, error) { + // create mock client and mock event recorder + // kbagent client is running in dry-run mode by setting context key-value pair: dry-run=true + store := newChangeCaptureStore(g.scheme, g.formatter) + mClient, err := newMockClient(g.cli, store, getKBOwnershipRules()) + if err != nil { + return nil, err + } + mEventRecorder := newMockEventRecorder(store) + + // build reconciler tree based on ownership rules: + // 1. each gvk has a corresponding reconciler + // 2. mock K8s native object reconciler + // 3. encapsulate KB controller as reconciler + reconcilerTree, err := newReconcilerTree(g.ctx, mClient, mEventRecorder, getKBOwnershipRules()) + if err != nil { + return nil, err + } + + // load current object tree into store + if err = loadCurrentObjectTree(g.loader, store); err != nil { + return nil, err + } + initialObjectMap := store.GetAll() + + // get current root + currentRoot := &kbappsv1.Cluster{} + if err = mClient.Get(g.ctx, client.ObjectKeyFromObject(desiredRoot), currentRoot); err != nil { + return nil, err + } + // build spec diff + var specDiff string + if specDiff, err = buildSpecDiff(currentRoot, desiredRoot); err != nil { + return nil, err + } + if err = mClient.Update(g.ctx, desiredRoot); err != nil { + return nil, err + } + + // generate plan with timeout + startTime := time.Now() + timeout := false + var reconcileErr error + previousCount := len(store.GetChanges()) + timeoutPeriod := 5 * time.Second + for { + if time.Since(startTime) > timeoutPeriod { + timeout = true + break + } + + // run reconciler tree + if reconcileErr = reconcilerTree.Run(); reconcileErr != nil { + break + } + + // no change (other than Events) means reconciliation cycle is done + currentCount := 0 + for _, change := range store.GetChanges() { + if change.ChangeType == tracev1.EventType { + continue + } + currentCount++ + } + if currentCount == previousCount { + break + } + previousCount = currentCount + } + + // update dry-run result + // update spec info + dryRunResult := &tracev1.DryRunResult{} + dryRunResult.ObservedTargetGeneration = currentRoot.Generation + dryRunResult.SpecDiff = specDiff + + // update phase + switch { + case reconcileErr != nil: + dryRunResult.Phase = tracev1.DryRunFailedPhase + dryRunResult.Reason = "ReconcileError" + dryRunResult.Message = reconcileErr.Error() + case timeout: + dryRunResult.Phase = tracev1.DryRunFailedPhase + dryRunResult.Reason = "Timeout" + dryRunResult.Message = fmt.Sprintf("Can't generate the plan within %d second", int(timeoutPeriod.Seconds())) + default: + dryRunResult.Phase = tracev1.DryRunSucceedPhase + } + + // update plan + desiredTree, err := getObjectTreeFromCache(g.ctx, mClient, desiredRoot, getKBOwnershipRules()) + if err != nil { + return nil, err + } + dryRunResult.Plan.ObjectTree = desiredTree + dryRunResult.Plan.Changes = store.GetChanges() + newObjectMap := store.GetAll() + dryRunResult.Plan.Summary.ObjectSummaries = buildObjectSummaries(initialObjectMap, newObjectMap) + + return dryRunResult, nil +} + +func newPlanGenerator(ctx context.Context, cli client.Client, scheme *runtime.Scheme, loader objectLoader, formatter descriptionFormatter) PlanGenerator { + return &planGenerator{ + ctx: ctx, + cli: cli, + scheme: scheme, + loader: loader, + formatter: formatter, + } +} + +func buildSpecDiff(current, desired client.Object) (string, error) { + // Extract the current spec + currentSpec, err := getFieldAsStruct(current, specFieldName) + if err != nil { + return "", err + } + desiredSpec, err := getFieldAsStruct(desired, specFieldName) + if err != nil { + return "", err + } + // build the spec change + specChange := cmp.Diff(currentSpec, desiredSpec) + return specChange, nil +} + +func loadCurrentObjectTree(loader objectLoader, store ChangeCaptureStore) error { + objectMap, err := loader() + if err != nil { + return err + } + for _, object := range objectMap { + if err := store.Load(object); err != nil { + return err + } + } + return nil +} + +var _ PlanGenerator = &planGenerator{} diff --git a/controllers/trace/reconciler_tree.go b/controllers/trace/reconciler_tree.go new file mode 100644 index 00000000000..608fc49732a --- /dev/null +++ b/controllers/trace/reconciler_tree.go @@ -0,0 +1,901 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + "fmt" + + vsv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v3/apis/volumesnapshot/v1beta1" + vsv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + "k8s.io/kubectl/pkg/util/podutils" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + workloadsAPI "github.com/apecloud/kubeblocks/apis/workloads/v1" + "github.com/apecloud/kubeblocks/controllers/apps" + "github.com/apecloud/kubeblocks/controllers/apps/configuration" + "github.com/apecloud/kubeblocks/controllers/dataprotection" + "github.com/apecloud/kubeblocks/controllers/workloads" + "github.com/apecloud/kubeblocks/pkg/constant" + "github.com/apecloud/kubeblocks/pkg/controller/builder" + "github.com/apecloud/kubeblocks/pkg/controller/graph" + "github.com/apecloud/kubeblocks/pkg/controller/model" + intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" + "github.com/apecloud/kubeblocks/pkg/dataprotection/types" +) + +type ReconcilerTree interface { + Run() error +} + +type reconcilerFunc func(client.Client, record.EventRecorder) reconcile.Reconciler + +var reconcilerFuncMap = map[tracev1.ObjectType]reconcilerFunc{ + objectType(kbappsv1.SchemeGroupVersion.String(), kbappsv1.ClusterKind): newClusterReconciler, + objectType(kbappsv1.SchemeGroupVersion.String(), kbappsv1.ComponentKind): newComponentReconciler, + objectType(corev1.SchemeGroupVersion.String(), constant.SecretKind): newSecretReconciler, + objectType(corev1.SchemeGroupVersion.String(), constant.ServiceKind): newServiceReconciler, + objectType(workloadsAPI.SchemeGroupVersion.String(), workloadsAPI.InstanceSetKind): newInstanceSetReconciler, + objectType(corev1.SchemeGroupVersion.String(), constant.ConfigMapKind): newConfigMapReconciler, + objectType(corev1.SchemeGroupVersion.String(), constant.PersistentVolumeClaimKind): newPVCReconciler, + objectType(rbacv1.SchemeGroupVersion.String(), constant.ClusterRoleBindingKind): newClusterRoleBindingReconciler, + objectType(rbacv1.SchemeGroupVersion.String(), constant.RoleBindingKind): newRoleBindingReconciler, + objectType(corev1.SchemeGroupVersion.String(), constant.ServiceAccountKind): newSAReconciler, + objectType(batchv1.SchemeGroupVersion.String(), constant.JobKind): newJobReconciler, + objectType(dpv1alpha1.SchemeGroupVersion.String(), types.BackupKind): newBackupReconciler, + objectType(dpv1alpha1.SchemeGroupVersion.String(), types.RestoreKind): newRestoreReconciler, + objectType(appsv1alpha1.SchemeGroupVersion.String(), constant.ConfigurationKind): newConfigurationReconciler, + objectType(corev1.SchemeGroupVersion.String(), constant.PodKind): newPodReconciler, + objectType(appsv1.SchemeGroupVersion.String(), constant.StatefulSetKind): newSTSReconciler, + objectType(vsv1.SchemeGroupVersion.String(), constant.VolumeSnapshotKind): newVolumeSnapshotV1Reconciler, + objectType(vsv1beta1.SchemeGroupVersion.String(), constant.VolumeSnapshotKind): newVolumeSnapshotV1Beta1Reconciler, + objectType(corev1.SchemeGroupVersion.String(), constant.PersistentVolumeKind): newPVReconciler, +} + +type reconcilerTree struct { + ctx context.Context + cli client.Client + tree *graph.DAG + reconcilers map[tracev1.ObjectType]reconcile.Reconciler +} + +func (r *reconcilerTree) Run() error { + return r.tree.WalkTopoOrder(func(v graph.Vertex) error { + objType, _ := v.(tracev1.ObjectType) + reconciler := r.reconcilers[objType] + gvk, err := objectTypeToGVK(&objType) + if err != nil { + return err + } + objects, err := getObjectsByGVK(r.ctx, r.cli, gvk, nil) + if err != nil { + return err + } + for _, object := range objects { + _, err = reconciler.Reconcile(r.ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(object)}) + if err != nil { + return err + } + } + return nil + }, func(v1, v2 graph.Vertex) bool { + t1, _ := v1.(tracev1.ObjectType) + t2, _ := v2.(tracev1.ObjectType) + if t1.APIVersion != t2.APIVersion { + return t1.APIVersion < t2.APIVersion + } + return t1.Kind < t2.Kind + }) +} + +func newReconcilerTree(ctx context.Context, mClient client.Client, recorder record.EventRecorder, rules []OwnershipRule) (ReconcilerTree, error) { + dag := graph.NewDAG() + reconcilers := make(map[tracev1.ObjectType]reconcile.Reconciler) + for _, rule := range rules { + dag.AddVertex(rule.Primary) + reconciler, err := newReconciler(mClient, recorder, rule.Primary) + if err != nil { + return nil, err + } + reconcilers[rule.Primary] = reconciler + for _, resource := range rule.OwnedResources { + dag.AddVertex(resource.Secondary) + dag.Connect(rule.Primary, resource.Secondary) + reconciler, err = newReconciler(mClient, recorder, resource.Secondary) + if err != nil { + return nil, err + } + reconcilers[resource.Secondary] = reconciler + } + } + // DAG should be valid(one and only one root without cycle) + if err := dag.Validate(); err != nil { + return nil, err + } + + return &reconcilerTree{ + ctx: ctx, + cli: mClient, + tree: dag, + reconcilers: reconcilers, + }, nil +} + +func newReconciler(mClient client.Client, recorder record.EventRecorder, objectType tracev1.ObjectType) (reconcile.Reconciler, error) { + reconcilerF, ok := reconcilerFuncMap[objectType] + if ok { + return reconcilerF(mClient, recorder), nil + } + return nil, fmt.Errorf("can't initialize a reconciler for GVK: %s/%s", objectType.APIVersion, objectType.Kind) +} + +func newClusterReconciler(cli client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return &apps.ClusterReconciler{ + Client: cli, + Scheme: cli.Scheme(), + Recorder: recorder, + } +} + +func newComponentReconciler(cli client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return &apps.ComponentReconciler{ + Client: cli, + Scheme: cli.Scheme(), + Recorder: recorder, + } +} + +func newConfigurationReconciler(cli client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return &configuration.ConfigurationReconciler{ + Client: cli, + Scheme: cli.Scheme(), + Recorder: recorder, + } +} + +func newInstanceSetReconciler(cli client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return &workloads.InstanceSetReconciler{ + Client: cli, + Scheme: cli.Scheme(), + Recorder: recorder, + } +} + +type baseReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder +} + +type doNothingReconciler struct { + baseReconciler +} + +func (r *doNothingReconciler) Reconcile(_ context.Context, _ reconcile.Request) (reconcile.Result, error) { + return reconcile.Result{}, nil +} + +type pvcReconciler struct { + baseReconciler +} + +func (r *pvcReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + pvc := &corev1.PersistentVolumeClaim{} + err := r.Get(ctx, req.NamespacedName, pvc) + if err != nil { + return reconcile.Result{}, client.IgnoreNotFound(err) + } + + // add finalizer + if len(pvc.Finalizers) == 0 { + pvc.Finalizers = []string{"kubernetes.io/pvc-protection"} + if err = r.Update(ctx, pvc); err != nil { + return reconcile.Result{}, err + } + } + + // handle deletion + if model.IsObjectDeleting(pvc) { + // delete pv + objKey := client.ObjectKey{ + Namespace: pvc.Namespace, + Name: pvc.Spec.VolumeName, + } + pv := &corev1.PersistentVolume{} + if err = r.Get(ctx, objKey, pv); err != nil && !apierrors.IsNotFound(err) { + return reconcile.Result{}, err + } else if err == nil { + if err = r.Delete(ctx, pv); err != nil { + return reconcile.Result{}, err + } + } + // remove finalizer + pvc.Finalizers = nil + if err = r.Update(ctx, pvc); err != nil { + return reconcile.Result{}, err + } + return reconcile.Result{}, r.Delete(ctx, pvc) + } + + // phase from "" to Pending + if pvc.Status.Phase == "" { + pvc.Status.Phase = corev1.ClaimPending + err = r.Status().Update(ctx, pvc) + return reconcile.Result{}, err + } + + // from Pending to Bound + if pvc.Status.Phase == corev1.ClaimPending { + // Provisioning PV for PVC + pvName := pvc.Name + "-pv" + pvcRef, err := getObjectReference(pvc, r.Scheme) + if err != nil { + return reconcile.Result{}, err + } + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvName, + }, + Spec: corev1.PersistentVolumeSpec{ + Capacity: corev1.ResourceList{ + corev1.ResourceStorage: pvc.Spec.Resources.Requests[corev1.ResourceStorage], + }, + AccessModes: pvc.Spec.AccessModes, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + StorageClassName: "mock-storage-class", + ClaimRef: pvcRef, + }, + } + + if err := r.Create(ctx, pv); err != nil { + return reconcile.Result{}, err + } + + pvc.Spec.VolumeName = pvName + pvc.Status.Phase = corev1.ClaimBound + if err = r.Status().Update(ctx, pvc); err != nil { + return reconcile.Result{}, err + } + } + + return reconcile.Result{}, nil +} + +func newPVCReconciler(c client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return &pvcReconciler{ + baseReconciler{ + Client: c, + Scheme: c.Scheme(), + Recorder: recorder, + }, + } +} + +type pvReconciler struct { + baseReconciler +} + +func (r *pvReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + pv := &corev1.PersistentVolume{} + err := r.Get(ctx, req.NamespacedName, pv) + if err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if model.IsObjectDeleting(pv) { + return reconcile.Result{}, r.Delete(ctx, pv) + } + + // phase from "" to Available + if pv.Status.Phase == "" { + pv.Status.Phase = corev1.VolumeAvailable + return reconcile.Result{}, r.Status().Update(ctx, pv) + } + + // phase from Available to Bound + if pv.Status.Phase == corev1.VolumeAvailable { + pv.Status.Phase = corev1.VolumeBound + return reconcile.Result{}, r.Status().Update(ctx, pv) + } + + // Delete PV if it's released + if pv.Status.Phase == corev1.VolumeReleased { + return ctrl.Result{}, r.Delete(ctx, pv) + } + + return ctrl.Result{}, nil +} + +func newPVReconciler(c client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return &pvReconciler{ + baseReconciler{ + Client: c, + Scheme: c.Scheme(), + Recorder: recorder, + }, + } +} + +type volumeSnapshotV1Beta1Reconciler struct { + baseReconciler +} + +func (r *volumeSnapshotV1Beta1Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + snapshot := &vsv1beta1.VolumeSnapshot{} + err := r.Get(ctx, req.NamespacedName, snapshot) + if err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // snapshot creation + if snapshot.Status == nil { + snapshot.Status = &vsv1beta1.VolumeSnapshotStatus{ + ReadyToUse: pointer.Bool(false), + } + if err = r.Status().Update(ctx, snapshot); err != nil { + return ctrl.Result{}, err + } + + // Update snapshot to Ready state + snapshot.Status.ReadyToUse = pointer.Bool(true) + snapshot.Status.Error = nil // Reset any errors + + if err = r.Status().Update(ctx, snapshot); err != nil { + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, nil +} + +func newVolumeSnapshotV1Beta1Reconciler(c client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return &volumeSnapshotV1Beta1Reconciler{ + baseReconciler{ + Client: c, + Scheme: c.Scheme(), + Recorder: recorder, + }, + } +} + +type volumeSnapshotV1Reconciler struct { + baseReconciler +} + +func (r *volumeSnapshotV1Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + snapshot := &vsv1.VolumeSnapshot{} + err := r.Get(ctx, req.NamespacedName, snapshot) + if err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // snapshot creation + if snapshot.Status == nil { + snapshot.Status = &vsv1.VolumeSnapshotStatus{ + ReadyToUse: pointer.Bool(false), + } + if err = r.Status().Update(ctx, snapshot); err != nil { + return ctrl.Result{}, err + } + // Update the snapshot to indicate it's ready + snapshot.Status.ReadyToUse = pointer.Bool(true) + snapshot.Status.Error = nil // No errors encountered + if err = r.Status().Update(ctx, snapshot); err != nil { + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, nil +} + +func newVolumeSnapshotV1Reconciler(c client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return &volumeSnapshotV1Reconciler{ + baseReconciler{ + Client: c, + Scheme: c.Scheme(), + Recorder: recorder, + }, + } +} + +type stsReconciler struct { + baseReconciler +} + +func (r *stsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + sts := &appsv1.StatefulSet{} + err := r.Get(ctx, req.NamespacedName, sts) + if err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + generatePodName := func(parent string, ordinal int32) string { + return fmt.Sprintf("%s-%d", parent, ordinal) + } + generatePVCName := func(claimTemplateName, podName string) string { + return fmt.Sprintf("%s-%s", claimTemplateName, podName) + } + + // handle deletion + if model.IsObjectDeleting(sts) { + // delete all pods + podList := &corev1.PodList{} + labels, err := metav1.LabelSelectorAsMap(sts.Spec.Selector) + if err != nil { + return ctrl.Result{}, err + } + if err = r.List(ctx, podList, client.MatchingLabels(labels)); err != nil { + return ctrl.Result{}, err + } + for _, pod := range podList.Items { + if err = r.Delete(ctx, &pod); err != nil { + return ctrl.Result{}, err + } + } + // delete sts + return ctrl.Result{}, r.Delete(ctx, sts) + } + + // handle creation + // Create Pods and PVCs for each replica + for i := int32(0); i < *sts.Spec.Replicas; i++ { + podName := generatePodName(sts.Name, i) + template := sts.Spec.Template + // 1. build pod + pod := builder.NewPodBuilder(sts.Namespace, podName). + AddAnnotationsInMap(template.Annotations). + AddLabelsInMap(template.Labels). + SetPodSpec(*template.Spec.DeepCopy()). + GetObject() + + // 2. build pvcs from template + pvcMap := make(map[string]*corev1.PersistentVolumeClaim) + pvcNameMap := make(map[string]string) + for _, claimTemplate := range sts.Spec.VolumeClaimTemplates { + pvcName := generatePVCName(claimTemplate.Name, pod.GetName()) + pvc := builder.NewPVCBuilder(sts.Namespace, pvcName). + AddLabelsInMap(template.Labels). + SetSpec(*claimTemplate.Spec.DeepCopy()). + GetObject() + pvcMap[pvcName] = pvc + pvcNameMap[pvcName] = claimTemplate.Name + } + + // 3. update pod volumes + var pvcs []*corev1.PersistentVolumeClaim + var volumeList []corev1.Volume + for pvcName, pvc := range pvcMap { + pvcs = append(pvcs, pvc) + volume := builder.NewVolumeBuilder(pvcNameMap[pvcName]). + SetVolumeSource(corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvcName}, + }).GetObject() + volumeList = append(volumeList, *volume) + } + intctrlutil.MergeList(&volumeList, &pod.Spec.Volumes, func(item corev1.Volume) func(corev1.Volume) bool { + return func(v corev1.Volume) bool { + return v.Name == item.Name + } + }) + + if err = controllerutil.SetControllerReference(sts, pod, r.Scheme); err != nil { + return reconcile.Result{}, err + } + + // Create Pod + if err = r.Create(ctx, pod); err != nil && !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } + // Create PVC + for _, pvc := range pvcs { + if err = r.Create(ctx, pvc); err != nil && !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } + } + } + + // Update the status of the StatefulSet + podList := &corev1.PodList{} + labels, err := metav1.LabelSelectorAsMap(sts.Spec.Selector) + if err != nil { + return ctrl.Result{}, err + } + if err = r.List(ctx, podList, client.MatchingLabels(labels)); err != nil { + return ctrl.Result{}, err + } + isCreated := func(pod *corev1.Pod) bool { + return pod.Status.Phase != "" + } + isRunningAndReady := func(pod *corev1.Pod) bool { + return pod.Status.Phase == corev1.PodRunning && podutils.IsPodReady(pod) + } + isTerminating := func(pod *corev1.Pod) bool { + return pod.DeletionTimestamp != nil + } + isRunningAndAvailable := func(pod *corev1.Pod, minReadySeconds int32) bool { + return podutils.IsPodAvailable(pod, minReadySeconds, metav1.Now()) + } + replicas := int32(0) + currentReplicas, updatedReplicas := int32(0), int32(0) + readyReplicas, availableReplicas := int32(0), int32(0) + for i := range podList.Items { + pod := &podList.Items[i] + if isCreated(pod) { + replicas++ + } + if isRunningAndReady(pod) && !isTerminating(pod) { + readyReplicas++ + if isRunningAndAvailable(pod, sts.Spec.MinReadySeconds) { + availableReplicas++ + } + } + if isCreated(pod) && !isTerminating(pod) { + updatedReplicas++ + } + } + sts.Status.Replicas = replicas + sts.Status.ReadyReplicas = readyReplicas + sts.Status.AvailableReplicas = availableReplicas + sts.Status.CurrentReplicas = currentReplicas + sts.Status.UpdatedReplicas = updatedReplicas + totalReplicas := int32(1) + if sts.Spec.Replicas != nil { + totalReplicas = *sts.Spec.Replicas + } + if sts.Status.Replicas == totalReplicas && sts.Status.UpdatedReplicas == totalReplicas { + sts.Status.CurrentRevision = sts.Status.UpdateRevision + sts.Status.CurrentReplicas = totalReplicas + } + return ctrl.Result{}, r.Status().Update(ctx, sts) +} + +func newSTSReconciler(c client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return &stsReconciler{ + baseReconciler{ + Client: c, + Scheme: c.Scheme(), + Recorder: recorder, + }, + } +} + +func newRestoreReconciler(c client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return &dataprotection.RestoreReconciler{ + Client: c, + Scheme: c.Scheme(), + Recorder: recorder, + } +} + +func newBackupReconciler(c client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return &dataprotection.BackupReconciler{ + Client: c, + Scheme: c.Scheme(), + Recorder: recorder, + } +} + +type jobReconciler struct { + baseReconciler +} + +func (r *jobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + job := &batchv1.Job{} + err := r.Get(ctx, req.NamespacedName, job) + if err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // handle deletion + if model.IsObjectDeleting(job) { + for i := int32(0); i < *job.Spec.Completions; i++ { + podName := fmt.Sprintf("%s-%d", job.Name, i) + pod := &corev1.Pod{} + if err = r.Get(ctx, client.ObjectKey{Namespace: job.Namespace, Name: podName}, pod); err == nil { + if err = r.Delete(ctx, pod); err != nil { + return ctrl.Result{}, err + } + } + } + return ctrl.Result{}, r.Delete(ctx, job) + } + + // Create Pods for the Job + number := int32(1) + if job.Spec.Parallelism != nil { + number = *job.Spec.Parallelism + } + for i := int32(0); i < number; i++ { + podName := fmt.Sprintf("%s-%d", job.Name, i) + pod := builder.NewPodBuilder(job.Namespace, podName). + AddLabelsInMap(job.Spec.Template.Labels). + SetPodSpec(*job.Spec.Template.Spec.DeepCopy()). + GetObject() + if err = controllerutil.SetControllerReference(job, pod, r.Scheme); err != nil { + return ctrl.Result{}, err + } + // Create Pod if not exists + if err = r.Create(ctx, pod); err != nil && !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } + // Update job status based on pod completion + if err = r.Get(ctx, client.ObjectKeyFromObject(pod), pod); err == nil { + if pod.Status.Phase == corev1.PodSucceeded { + job.Status.Succeeded++ + } else if pod.Status.Phase == corev1.PodFailed { + job.Status.Failed++ + } + } + } + + // Update the job status + if err = r.Status().Update(ctx, job); err != nil { + return ctrl.Result{}, err + } + + // Cleanup logic: delete pods if job has succeeded or failed + if job.Status.Succeeded == number || job.Status.Failed > 0 { + for i := int32(0); i < number; i++ { + podName := fmt.Sprintf("%s-%d", job.Name, i) + pod := &corev1.Pod{} + if err = r.Get(ctx, client.ObjectKey{Namespace: job.Namespace, Name: podName}, pod); err == nil { + if err = r.Delete(ctx, pod); err != nil { + return ctrl.Result{}, err + } + } + } + } + + return ctrl.Result{}, nil +} + +func newJobReconciler(c client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return &jobReconciler{ + baseReconciler{ + Client: c, + Scheme: c.Scheme(), + Recorder: recorder, + }, + } +} + +func newSAReconciler(c client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return newDoNothingReconciler() +} + +func newRoleBindingReconciler(c client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return newDoNothingReconciler() +} + +func newClusterRoleBindingReconciler(c client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return newDoNothingReconciler() +} + +func newConfigMapReconciler(c client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return newDoNothingReconciler() +} + +func newSecretReconciler(c client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return newDoNothingReconciler() +} + +func newDoNothingReconciler() reconcile.Reconciler { + return &doNothingReconciler{} +} + +type podReconciler struct { + baseReconciler +} + +func (r *podReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + pod := &corev1.Pod{} + err := r.Get(ctx, req.NamespacedName, pod) + if err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // handle deletion + if model.IsObjectDeleting(pod) { + return reconcile.Result{}, r.Delete(ctx, pod) + } + + if pod.Status.Phase == corev1.PodRunning { + return reconcile.Result{}, nil + } + + // phase from "" to Pending + if pod.Status.Phase == "" { + pod.Status.Phase = corev1.PodPending + if err = r.Status().Update(ctx, pod); err != nil { + return reconcile.Result{}, err + } + } + + // phase from Pending to ContainerCreating + // transition to PodScheduled + // Check if the PodScheduled condition exists + podScheduled := false + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodScheduled { + podScheduled = true + } + } + + if !podScheduled { + newCondition := corev1.PodCondition{ + Type: corev1.PodScheduled, + Status: corev1.ConditionTrue, + Reason: "PodScheduled", + Message: "Pod has been scheduled successfully.", + LastTransitionTime: metav1.Now(), + } + pod.Status.Conditions = append(pod.Status.Conditions, newCondition) + if err = r.Status().Update(ctx, pod); err != nil { + return reconcile.Result{}, err + } + } + + // Transition to ContainerCreating + // check if the PodReadyToStartContainers condition exists + podReadyToStart := false + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReadyToStartContainers { + podReadyToStart = true + } + } + if !podReadyToStart { + conditions := []corev1.PodCondition{ + { + Type: corev1.PodInitialized, + Status: corev1.ConditionTrue, + LastTransitionTime: metav1.Now(), + }, + { + Type: corev1.PodReadyToStartContainers, + Status: corev1.ConditionFalse, + LastTransitionTime: metav1.Now(), + }, + { + Type: corev1.ContainersReady, + Status: corev1.ConditionFalse, + Reason: "ContainersNotReady", + Message: "containers with unready status", + LastTransitionTime: metav1.Now(), + }, + { + Type: corev1.PodReady, + Status: corev1.ConditionFalse, + Reason: "ContainersNotReady", + Message: "containers with unready status", + LastTransitionTime: metav1.Now(), + }, + } + pod.Status.Conditions = append(pod.Status.Conditions, conditions...) + // containers status to Waiting + var containerStatuses []corev1.ContainerStatus + for _, container := range pod.Spec.Containers { + containerStatus := corev1.ContainerStatus{ + Image: container.Image, + ImageID: "", + Name: container.Name, + Ready: false, + Started: pointer.Bool(false), + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Reason: "ContainerCreating", + }, + }, + } + containerStatuses = append(containerStatuses, containerStatus) + } + pod.Status.ContainerStatuses = containerStatuses + if err = r.Status().Update(ctx, pod); err != nil { + return reconcile.Result{}, err + } + } + + // Transition to ContainerReady + // Check ContainerReady condition + containerReady := false + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.ContainersReady && condition.Status == corev1.ConditionTrue { + containerReady = true + } + } + if !containerReady { + for i := range pod.Status.Conditions { + cond := &pod.Status.Conditions[i] + cond.Status = corev1.ConditionTrue + cond.Reason = "" + cond.Message = "" + } + for i := range pod.Status.ContainerStatuses { + status := &pod.Status.ContainerStatuses[i] + status.Ready = true + status.Started = pointer.Bool(true) + status.State = corev1.ContainerState{ + Running: &corev1.ContainerStateRunning{ + StartedAt: metav1.Now(), + }, + } + } + pod.Status.Phase = corev1.PodRunning + if err = r.Status().Update(ctx, pod); err != nil { + return reconcile.Result{}, err + } + } + + isJobPod := false + for _, reference := range pod.OwnerReferences { + if reference.Kind == "Job" { + isJobPod = true + } + } + if !isJobPod { + return reconcile.Result{}, err + } + if pod.Status.Phase != corev1.PodSucceeded { + pod.Status.Phase = corev1.PodSucceeded + if err = r.Status().Update(ctx, pod); err != nil { + return reconcile.Result{}, err + } + } + + return ctrl.Result{}, nil +} + +func newPodReconciler(cli client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return &podReconciler{ + baseReconciler: baseReconciler{ + Client: cli, + Scheme: cli.Scheme(), + Recorder: recorder, + }, + } +} + +func newServiceReconciler(cli client.Client, recorder record.EventRecorder) reconcile.Reconciler { + return newDoNothingReconciler() +} + +var _ ReconcilerTree = &reconcilerTree{} +var _ reconcile.Reconciler = &doNothingReconciler{} +var _ reconcile.Reconciler = &podReconciler{} +var _ reconcile.Reconciler = &pvcReconciler{} +var _ reconcile.Reconciler = &pvReconciler{} +var _ reconcile.Reconciler = &volumeSnapshotV1Beta1Reconciler{} +var _ reconcile.Reconciler = &volumeSnapshotV1Reconciler{} +var _ reconcile.Reconciler = &stsReconciler{} diff --git a/controllers/trace/reconciler_tree_test.go b/controllers/trace/reconciler_tree_test.go new file mode 100644 index 00000000000..3f487a82136 --- /dev/null +++ b/controllers/trace/reconciler_tree_test.go @@ -0,0 +1,213 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/golang/mock/gomock" + vsv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/apecloud/kubeblocks/pkg/constant" + "github.com/apecloud/kubeblocks/pkg/controller/builder" + testutil "github.com/apecloud/kubeblocks/pkg/testutil/k8s" + "github.com/apecloud/kubeblocks/pkg/testutil/k8s/mocks" +) + +var _ = Describe("reconciler_tree test", func() { + var ( + k8sMock *mocks.MockClient + controller *gomock.Controller + ) + + BeforeEach(func() { + controller, k8sMock = testutil.SetupK8sMock() + }) + + AfterEach(func() { + controller.Finish() + }) + + Context("Testing reconciler_tree", func() { + var ( + mClient client.Client + mRecorder record.EventRecorder + reconcilerTree ReconcilerTree + ) + + reconcileN := func(n int) error { + for i := 0; i < n; i++ { + if err := reconcilerTree.Run(); err != nil { + return err + } + } + return nil + } + + BeforeEach(func() { + i18n := builder.NewConfigMapBuilder(namespace, name).SetData( + map[string]string{"en": "apps.kubeblocks.io/v1/Component/Creation=Component %s/%s is created."}, + ).GetObject() + store := newChangeCaptureStore(scheme.Scheme, buildDescriptionFormatter(i18n, defaultLocale, nil)) + k8sMock.EXPECT().Scheme().Return(scheme.Scheme).AnyTimes() + var err error + mClient, err = newMockClient(k8sMock, store, getKBOwnershipRules()) + Expect(err).ToNot(HaveOccurred()) + mRecorder = newMockEventRecorder(store) + + reconcilerTree, err = newReconcilerTree(ctx, mClient, mRecorder, getKBOwnershipRules()) + Expect(err).ToNot(HaveOccurred()) + }) + + It("reconcile with nothing", func() { + Expect(reconcileN(1)).Should(Succeed()) + }) + + It("reconcile a Pod", func() { + container := corev1.Container{ + Name: "test", + Image: "busybox", + } + pod := builder.NewPodBuilder(namespace, name).AddContainer(container).GetObject() + Expect(mClient.Create(ctx, pod)).Should(Succeed()) + Expect(reconcileN(10)).Should(Succeed()) + podRes := &corev1.Pod{} + Expect(mClient.Get(ctx, client.ObjectKeyFromObject(pod), podRes)).Should(Succeed()) + Expect(podRes.Status.Phase).Should(Equal(corev1.PodRunning)) + }) + + It("reconcile PVC&PV", func() { + resources := corev1.VolumeResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + } + pvc := builder.NewPVCBuilder(namespace, name).SetResources(resources).GetObject() + Expect(mClient.Create(ctx, pvc)).Should(Succeed()) + Expect(reconcileN(10)).Should(Succeed()) + pvcRes := &corev1.PersistentVolumeClaim{} + pvRes := &corev1.PersistentVolume{} + Expect(mClient.Get(ctx, client.ObjectKeyFromObject(pvc), pvcRes)).Should(Succeed()) + Expect(pvcRes.Status.Phase).Should(Equal(corev1.ClaimBound)) + pvKey := client.ObjectKey{Name: pvc.Name + "-pv"} + Expect(mClient.Get(ctx, pvKey, pvRes)).Should(Succeed()) + Expect(pvRes.Status.Phase).Should(Equal(corev1.VolumeBound)) + }) + + It("reconcile a Job", func() { + container := corev1.Container{ + Name: "test", + Image: "busybox", + } + pod := builder.NewPodBuilder(namespace, name).AddContainer(container).GetObject() + job := builder.NewJobBuilder(namespace, name).SetPodTemplateSpec(corev1.PodTemplateSpec{Spec: pod.Spec}).GetObject() + + By("create the Job") + Expect(mClient.Create(ctx, job)).Should(Succeed()) + + By("verify the Pod been created") + Expect(reconcileN(1)).Should(Succeed()) + key := client.ObjectKey{Namespace: job.Namespace, Name: job.Name + "-0"} + Expect(mClient.Get(ctx, key, &corev1.Pod{})).Should(Succeed()) + + By("pod succeed") + Expect(reconcileN(10)).Should(Succeed()) + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &corev1.Pod{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *corev1.Pod, _ ...client.GetOption) error { + return apierrors.NewNotFound(corev1.Resource(constant.PodKind), objKey.Name) + }).Times(1) + err := mClient.Get(ctx, key, &corev1.Pod{}) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + + By("job succeed") + Expect(mClient.Get(ctx, client.ObjectKeyFromObject(job), job)).Should(Succeed()) + Expect(job.Status.Succeeded).Should(BeEquivalentTo(1)) + }) + + It("reconcile a StatefulSet", func() { + container := corev1.Container{ + Name: "test", + Image: "busybox", + } + pod := builder.NewPodBuilder(namespace, name).AddLabels("hello", "world").AddContainer(container).GetObject() + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "hello": "world", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: pod.ObjectMeta, + Spec: pod.Spec, + }, + Replicas: pointer.Int32(1), + }, + } + Expect(mClient.Create(ctx, sts)).Should(Succeed()) + Expect(reconcileN(10)).Should(Succeed()) + Expect(mClient.Get(ctx, client.ObjectKeyFromObject(sts), sts)).Should(Succeed()) + Expect(sts.Status.ReadyReplicas).Should(BeEquivalentTo(*sts.Spec.Replicas)) + }) + + It("reconcile VolumeSnapshot v1", func() { + reconciler := newVolumeSnapshotV1Reconciler(mClient, mRecorder) + v1 := &vsv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + } + Expect(mClient.Create(ctx, v1)).Should(Succeed()) + + key := client.ObjectKeyFromObject(v1) + for i := 0; i < 10; i++ { + _, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + Expect(err).ToNot(HaveOccurred()) + } + Expect(mClient.Get(ctx, key, v1)).Should(Succeed()) + Expect(v1.Status).ShouldNot(BeNil()) + Expect(v1.Status.ReadyToUse).ShouldNot(BeNil()) + Expect(*v1.Status.ReadyToUse).Should(BeTrue()) + }) + }) +}) diff --git a/controllers/trace/reconciliationtrace_controller.go b/controllers/trace/reconciliationtrace_controller.go new file mode 100644 index 00000000000..1925ba00331 --- /dev/null +++ b/controllers/trace/reconciliationtrace_controller.go @@ -0,0 +1,86 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/source" + + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx" + "github.com/apecloud/kubeblocks/pkg/controller/model" +) + +func init() { + model.AddScheme(tracev1.AddToScheme) +} + +// ReconciliationTraceReconciler reconciles a ReconciliationTrace object +type ReconciliationTraceReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + ObjectRevisionStore ObjectRevisionStore + ObjectTreeRootFinder ObjectTreeRootFinder + InformerManager InformerManager +} + +//+kubebuilder:rbac:groups=trace.kubeblocks.io,resources=reconciliationtraces,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=trace.kubeblocks.io,resources=reconciliationtraces/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=trace.kubeblocks.io,resources=reconciliationtraces/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.14.4/pkg/reconcile +func (r *ReconciliationTraceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx).WithValues("ReconciliationTrace", req.NamespacedName) + + res, err := kubebuilderx.NewController(ctx, r.Client, req, r.Recorder, logger). + Prepare(traceResources()). + Do(resourcesValidation(ctx, r.Client)). + Do(assureFinalizer()). + Do(handleDeletion(r.ObjectRevisionStore)). + Do(dryRun(ctx, r.Client, r.Scheme)). + Do(updateCurrentState(ctx, r.Client, r.Scheme, r.ObjectRevisionStore)). + Do(updateDesiredState(ctx, r.Client, r.Scheme, r.ObjectRevisionStore)). + Commit() + + return res, err +} + +// SetupWithManager sets up the controller with the Manager. +func (r *ReconciliationTraceReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.ObjectRevisionStore = NewObjectStore(r.Scheme) + r.ObjectTreeRootFinder = NewObjectTreeRootFinder(r.Client) + r.InformerManager = NewInformerManager(r.Client, mgr.GetCache(), r.Scheme, r.ObjectTreeRootFinder.GetEventChannel()) + + return ctrl.NewControllerManagedBy(mgr). + For(&tracev1.ReconciliationTrace{}). + WatchesRawSource(&source.Channel{Source: r.ObjectTreeRootFinder.GetEventChannel()}, r.ObjectTreeRootFinder.GetEventHandler()). + Complete(r) +} diff --git a/controllers/trace/resources_loader.go b/controllers/trace/resources_loader.go new file mode 100644 index 00000000000..899b25bed81 --- /dev/null +++ b/controllers/trace/resources_loader.go @@ -0,0 +1,73 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/constant" + "github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx" + viper "github.com/apecloud/kubeblocks/pkg/viperx" +) + +type resourcesLoader struct{} + +func (r *resourcesLoader) Load(ctx context.Context, reader client.Reader, req ctrl.Request, recorder record.EventRecorder, logger logr.Logger) (*kubebuilderx.ObjectTree, error) { + // load trace object + tree, err := kubebuilderx.ReadObjectTree[*tracev1.ReconciliationTrace](ctx, reader, req, nil) + if err != nil { + return nil, err + } + if tree.GetRoot() == nil { + return tree, nil + } + + // load i18n resources + i18n := &corev1.ConfigMap{} + i18nResourcesNamespace := viper.GetString(constant.CfgKeyCtrlrMgrNS) + i18nResourcesName := viper.GetString(constant.I18nResourcesName) + if err = reader.Get(ctx, types.NamespacedName{Namespace: i18nResourcesNamespace, Name: i18nResourcesName}, i18n); err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + if err = tree.Add(i18n); err != nil { + return nil, err + } + + tree.EventRecorder = recorder + tree.Logger = logger + tree.SetFinalizer(finalizer) + + return tree, nil +} + +func traceResources() kubebuilderx.TreeLoader { + return &resourcesLoader{} +} + +var _ kubebuilderx.TreeLoader = &resourcesLoader{} diff --git a/controllers/trace/resources_loader_test.go b/controllers/trace/resources_loader_test.go new file mode 100644 index 00000000000..b9f3718c53c --- /dev/null +++ b/controllers/trace/resources_loader_test.go @@ -0,0 +1,84 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/golang/mock/gomock" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/constant" + testutil "github.com/apecloud/kubeblocks/pkg/testutil/k8s" + "github.com/apecloud/kubeblocks/pkg/testutil/k8s/mocks" +) + +var _ = Describe("resources_loader test", func() { + var ( + k8sMock *mocks.MockClient + controller *gomock.Controller + ) + + BeforeEach(func() { + controller, k8sMock = testutil.SetupK8sMock() + }) + + AfterEach(func() { + controller.Finish() + }) + + Context("Testing resources_loader", func() { + It("should work well", func() { + trace := &tracev1.ReconciliationTrace{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + } + + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &tracev1.ReconciliationTrace{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *tracev1.ReconciliationTrace, _ ...client.GetOption) error { + *obj = *trace + return nil + }).Times(1) + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &corev1.ConfigMap{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *corev1.ConfigMap, _ ...client.GetOption) error { + return apierrors.NewNotFound(corev1.Resource(constant.ConfigMapKind), objKey.Name) + }).Times(1) + + loader := traceResources() + logger := log.FromContext(ctx).WithName("resources_loader test") + tree, err := loader.Load(ctx, k8sMock, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(trace)}, nil, logger) + Expect(err).ToNot(HaveOccurred()) + Expect(tree.GetRoot()).To(Equal(trace)) + }) + }) +}) diff --git a/controllers/trace/resources_validator.go b/controllers/trace/resources_validator.go new file mode 100644 index 00000000000..dd12b9a1d54 --- /dev/null +++ b/controllers/trace/resources_validator.go @@ -0,0 +1,68 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" + + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx" +) + +type resourcesValidator struct { + ctx context.Context + reader client.Reader +} + +func (r *resourcesValidator) PreCondition(tree *kubebuilderx.ObjectTree) *kubebuilderx.CheckResult { + if tree == nil { + return kubebuilderx.ConditionUnsatisfied + } + return kubebuilderx.ConditionSatisfied +} + +func (r *resourcesValidator) Reconcile(tree *kubebuilderx.ObjectTree) (kubebuilderx.Result, error) { + // trace object should exist + if tree.GetRoot() == nil { + return kubebuilderx.Commit, nil + } + + // target object should exist + v, _ := tree.GetRoot().(*tracev1.ReconciliationTrace) + objectKey := client.ObjectKeyFromObject(v) + if v.Spec.TargetObject != nil { + objectKey.Namespace = v.Spec.TargetObject.Namespace + objectKey.Name = v.Spec.TargetObject.Name + } + if err := r.reader.Get(r.ctx, objectKey, &kbappsv1.Cluster{}); err != nil { + return kubebuilderx.Commit, err + } + + return kubebuilderx.Continue, nil +} + +func resourcesValidation(ctx context.Context, reader client.Reader) kubebuilderx.Reconciler { + return &resourcesValidator{ctx: ctx, reader: reader} +} + +var _ kubebuilderx.Reconciler = &resourcesValidator{} diff --git a/controllers/trace/resources_validator_test.go b/controllers/trace/resources_validator_test.go new file mode 100644 index 00000000000..3ee740949b8 --- /dev/null +++ b/controllers/trace/resources_validator_test.go @@ -0,0 +1,81 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/golang/mock/gomock" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/controller/builder" + "github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx" + testutil "github.com/apecloud/kubeblocks/pkg/testutil/k8s" + "github.com/apecloud/kubeblocks/pkg/testutil/k8s/mocks" +) + +var _ = Describe("resources_loader test", func() { + var ( + k8sMock *mocks.MockClient + controller *gomock.Controller + ) + + BeforeEach(func() { + controller, k8sMock = testutil.SetupK8sMock() + }) + + AfterEach(func() { + controller.Finish() + }) + + Context("Testing resources_loader", func() { + It("should work well", func() { + trace := &tracev1.ReconciliationTrace{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + } + target := builder.NewClusterBuilder(namespace, name).GetObject() + + k8sMock.EXPECT(). + Get(gomock.Any(), gomock.Any(), &kbappsv1.Cluster{}, gomock.Any()). + DoAndReturn(func(_ context.Context, objKey client.ObjectKey, obj *kbappsv1.Cluster, _ ...client.GetOption) error { + *obj = *target + return nil + }).Times(1) + + tree := kubebuilderx.NewObjectTree() + tree.SetRoot(trace) + + reconciler := resourcesValidation(ctx, k8sMock) + Expect(reconciler.PreCondition(tree)).To(Equal(kubebuilderx.ConditionSatisfied)) + res, err := reconciler.Reconcile(tree) + Expect(err).ToNot(HaveOccurred()) + Expect(res).To(Equal(kubebuilderx.Continue)) + }) + }) +}) diff --git a/controllers/trace/suite_test.go b/controllers/trace/suite_test.go new file mode 100644 index 00000000000..4ae6be02890 --- /dev/null +++ b/controllers/trace/suite_test.go @@ -0,0 +1,192 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/golang/mock/gomock" + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + appsv1beta1 "github.com/apecloud/kubeblocks/apis/apps/v1beta1" + dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" + opsv1alpha1 "github.com/apecloud/kubeblocks/apis/operations/v1alpha1" + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + workloadsv1 "github.com/apecloud/kubeblocks/apis/workloads/v1" + "github.com/apecloud/kubeblocks/pkg/controller/builder" + "github.com/apecloud/kubeblocks/pkg/controller/model" + "github.com/apecloud/kubeblocks/pkg/testutil/k8s/mocks" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + cfg *rest.Config + testEnv *envtest.Environment + ctx context.Context + cancel context.CancelFunc + + namespace = "foo" + name = "bar" + uid = uuid.NewUUID() + resourceVersion = "612345" +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + initKBOwnershipRulesForTest(cfg) + + err = appsv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + model.AddScheme(appsv1alpha1.AddToScheme) + + err = opsv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + model.AddScheme(opsv1alpha1.AddToScheme) + + err = appsv1beta1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + model.AddScheme(appsv1beta1.AddToScheme) + + err = kbappsv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + model.AddScheme(kbappsv1.AddToScheme) + + err = dpv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + model.AddScheme(dpv1alpha1.AddToScheme) + + err = snapshotv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + model.AddScheme(snapshotv1.AddToScheme) + + err = workloadsv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + model.AddScheme(workloadsv1.AddToScheme) + + err = tracev1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + model.AddScheme(tracev1.AddToScheme) + + go func() { + defer GinkgoRecover() + }() +}) + +var _ = AfterSuite(func() { + cancel() + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +func mockObjects(k8sMock *mocks.MockClient) (*kbappsv1.Cluster, []kbappsv1.Component) { + primary := builder.NewClusterBuilder(namespace, name).SetUID(uid).SetResourceVersion(resourceVersion).GetObject() + compNames := []string{"hello", "world"} + var secondaries []kbappsv1.Component + for _, compName := range compNames { + fullCompName := fmt.Sprintf("%s-%s", primary.Name, compName) + secondary := builder.NewComponentBuilder(namespace, fullCompName, ""). + SetOwnerReferences(kbappsv1.APIVersion, kbappsv1.ClusterKind, primary). + SetUID(uid). + GetObject() + secondary.ResourceVersion = resourceVersion + secondaries = append(secondaries, *secondary) + } + k8sMock.EXPECT().Scheme().Return(scheme.Scheme).AnyTimes() + k8sMock.EXPECT(). + List(gomock.Any(), &kbappsv1.ComponentList{}, gomock.Any()). + DoAndReturn(func(_ context.Context, list *kbappsv1.ComponentList, _ ...client.ListOption) error { + list.Items = secondaries + return nil + }).Times(1) + k8sMock.EXPECT(). + List(gomock.Any(), &corev1.ServiceList{}, gomock.Any()). + DoAndReturn(func(_ context.Context, list *corev1.ServiceList, _ ...client.ListOption) error { + return nil + }).Times(1) + k8sMock.EXPECT(). + List(gomock.Any(), &corev1.SecretList{}, gomock.Any()). + DoAndReturn(func(_ context.Context, list *corev1.SecretList, _ ...client.ListOption) error { + return nil + }).Times(1) + componentSecondaries := []client.ObjectList{ + &workloadsv1.InstanceSetList{}, + &corev1.ServiceList{}, + &corev1.SecretList{}, + &corev1.ConfigMapList{}, + &corev1.PersistentVolumeClaimList{}, + &rbacv1.ClusterRoleBindingList{}, + &rbacv1.RoleBindingList{}, + &corev1.ServiceAccountList{}, + &batchv1.JobList{}, + &dpv1alpha1.BackupList{}, + &dpv1alpha1.RestoreList{}, + &appsv1alpha1.ConfigurationList{}, + } + for _, secondary := range componentSecondaries { + k8sMock.EXPECT(). + List(gomock.Any(), secondary, gomock.Any()). + DoAndReturn(func(_ context.Context, list client.ObjectList, _ ...client.ListOption) error { + return nil + }).Times(2) + } + return primary, secondaries +} diff --git a/controllers/trace/type.go b/controllers/trace/type.go new file mode 100644 index 00000000000..4c2d44a85b4 --- /dev/null +++ b/controllers/trace/type.go @@ -0,0 +1,453 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "fmt" + "sync" + + vsv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v3/apis/volumesnapshot/v1beta1" + vsv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/discovery" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + workloads "github.com/apecloud/kubeblocks/apis/workloads/v1" + "github.com/apecloud/kubeblocks/pkg/constant" + "github.com/apecloud/kubeblocks/pkg/controller/instanceset" + intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" + dptypes "github.com/apecloud/kubeblocks/pkg/dataprotection/types" +) + +const finalizer = "trace.kubeblocks.io/finalizer" + +const ( + specFieldName = "Spec" + statusFieldName = "Status" +) + +var ( + clusterCriteria = OwnershipCriteria{ + LabelCriteria: map[string]string{ + constant.AppInstanceLabelKey: "$(primary.name)", + constant.AppManagedByLabelKey: constant.AppName, + }, + Validation: OwnerValidation, + } + + componentCriteria = OwnershipCriteria{ + LabelCriteria: map[string]string{ + constant.AppInstanceLabelKey: "$(primary)", + constant.AppManagedByLabelKey: constant.AppName, + }, + Validation: OwnerValidation, + } + + itsCriteria = OwnershipCriteria{ + LabelCriteria: map[string]string{ + instanceset.WorkloadsManagedByLabelKey: workloads.InstanceSetKind, + instanceset.WorkloadsInstanceLabelKey: "$(primary.name)", + }, + } + + configurationCriteria = componentCriteria + + backupCriteria = OwnershipCriteria{ + LabelCriteria: map[string]string{ + constant.AppInstanceLabelKey: "$(primary)", + constant.AppManagedByLabelKey: dptypes.AppName, + }, + } + + restoreCriteria = backupCriteria + + pvcCriteria = OwnershipCriteria{ + SpecifiedNameCriteria: &FieldPath{ + Path: "spec.volumeName", + }, + } + + fullKBOwnershipRules = []OwnershipRule{ + { + Primary: objectType(kbappsv1.SchemeGroupVersion.String(), kbappsv1.ClusterKind), + OwnedResources: []OwnedResource{ + { + Secondary: objectType(kbappsv1.SchemeGroupVersion.String(), kbappsv1.ComponentKind), + Criteria: clusterCriteria, + }, + { + Secondary: objectType(corev1.SchemeGroupVersion.String(), constant.ServiceKind), + Criteria: clusterCriteria, + }, + { + Secondary: objectType(corev1.SchemeGroupVersion.String(), constant.SecretKind), + Criteria: clusterCriteria, + }, + }, + }, + { + Primary: objectType(kbappsv1.SchemeGroupVersion.String(), kbappsv1.ComponentKind), + OwnedResources: []OwnedResource{ + { + Secondary: objectType(workloads.SchemeGroupVersion.String(), workloads.InstanceSetKind), + Criteria: componentCriteria, + }, + { + Secondary: objectType(corev1.SchemeGroupVersion.String(), constant.ServiceKind), + Criteria: componentCriteria, + }, + { + Secondary: objectType(corev1.SchemeGroupVersion.String(), constant.SecretKind), + Criteria: componentCriteria, + }, + { + Secondary: objectType(corev1.SchemeGroupVersion.String(), constant.ConfigMapKind), + Criteria: componentCriteria, + }, + { + Secondary: objectType(corev1.SchemeGroupVersion.String(), constant.PersistentVolumeClaimKind), + Criteria: componentCriteria, + }, + { + Secondary: objectType(rbacv1.SchemeGroupVersion.String(), constant.ClusterRoleBindingKind), + Criteria: componentCriteria, + }, + { + Secondary: objectType(rbacv1.SchemeGroupVersion.String(), constant.RoleBindingKind), + Criteria: componentCriteria, + }, + { + Secondary: objectType(corev1.SchemeGroupVersion.String(), constant.ServiceAccountKind), + Criteria: componentCriteria, + }, + { + Secondary: objectType(batchv1.SchemeGroupVersion.String(), constant.JobKind), + Criteria: componentCriteria, + }, + { + Secondary: objectType(dpv1alpha1.SchemeGroupVersion.String(), dptypes.BackupKind), + Criteria: componentCriteria, + }, + { + Secondary: objectType(dpv1alpha1.SchemeGroupVersion.String(), dptypes.RestoreKind), + Criteria: componentCriteria, + }, + { + Secondary: objectType(appsv1alpha1.SchemeGroupVersion.String(), constant.ConfigurationKind), + Criteria: componentCriteria, + }, + }, + }, + { + Primary: objectType(workloads.SchemeGroupVersion.String(), workloads.InstanceSetKind), + OwnedResources: []OwnedResource{ + { + Secondary: objectType(corev1.SchemeGroupVersion.String(), constant.PodKind), + Criteria: itsCriteria, + }, + { + Secondary: objectType(corev1.SchemeGroupVersion.String(), constant.ServiceKind), + Criteria: itsCriteria, + }, + { + Secondary: objectType(corev1.SchemeGroupVersion.String(), constant.PersistentVolumeClaimKind), + Criteria: itsCriteria, + }, + { + Secondary: objectType(corev1.SchemeGroupVersion.String(), constant.ConfigMapKind), + Criteria: itsCriteria, + }, + }, + }, + { + Primary: objectType(kbappsv1.SchemeGroupVersion.String(), constant.ConfigurationKind), + OwnedResources: []OwnedResource{ + { + Secondary: objectType(corev1.SchemeGroupVersion.String(), constant.ConfigMapKind), + Criteria: configurationCriteria, + }, + }, + }, + { + Primary: objectType(dpv1alpha1.SchemeGroupVersion.String(), dptypes.BackupKind), + OwnedResources: []OwnedResource{ + { + Secondary: objectType(batchv1.SchemeGroupVersion.String(), constant.JobKind), + Criteria: backupCriteria, + }, + { + Secondary: objectType(appsv1.SchemeGroupVersion.String(), constant.StatefulSetKind), + Criteria: backupCriteria, + }, + { + Secondary: objectType(vsv1.SchemeGroupVersion.String(), constant.VolumeSnapshotKind), + Criteria: backupCriteria, + }, + { + Secondary: objectType(vsv1beta1.SchemeGroupVersion.String(), constant.VolumeSnapshotKind), + Criteria: backupCriteria, + }, + }, + }, + { + Primary: objectType(dpv1alpha1.SchemeGroupVersion.String(), dptypes.RestoreKind), + OwnedResources: []OwnedResource{ + { + Secondary: objectType(batchv1.SchemeGroupVersion.String(), constant.JobKind), + Criteria: restoreCriteria, + }, + }, + }, + { + Primary: objectType(corev1.SchemeGroupVersion.String(), constant.PersistentVolumeClaimKind), + OwnedResources: []OwnedResource{ + { + Secondary: objectType(corev1.SchemeGroupVersion.String(), constant.PersistentVolumeKind), + Criteria: pvcCriteria, + }, + }, + }, + } + + kbOwnershipRules []OwnershipRule + once sync.Once +) + +func getKBOwnershipRules() []OwnershipRule { + once.Do(initKBOwnershipRules(nil)) + return kbOwnershipRules +} + +func initKBOwnershipRulesForTest(cfg *rest.Config) { + once.Do(initKBOwnershipRules(cfg)) +} + +func initKBOwnershipRules(cfg *rest.Config) func() { + return func() { + kbOwnershipRules = filterUnsupportedRules(fullKBOwnershipRules, cfg) + } +} + +func filterUnsupportedRules(ownershipRules []OwnershipRule, cfg *rest.Config) []OwnershipRule { + if cfg == nil { + cfg = intctrlutil.GetKubeRestConfig("kubeblocks-api-tester") + } + var rules []OwnershipRule + for _, rule := range ownershipRules { + if exists, _ := resourceExists(rule.Primary.APIVersion, rule.Primary.Kind, cfg); !exists { + continue + } + filteredRule := OwnershipRule{ + Primary: rule.Primary, + } + for _, ownedResource := range rule.OwnedResources { + if exists, _ := resourceExists(ownedResource.Secondary.APIVersion, ownedResource.Secondary.Kind, cfg); !exists { + continue + } + filteredRule.OwnedResources = append(filteredRule.OwnedResources, ownedResource) + } + if len(filteredRule.OwnedResources) > 0 { + rules = append(rules, filteredRule) + } + } + return rules +} + +// resourceExists checks if a resource with the given apiVersion and kind exists in the cluster. +func resourceExists(apiVersion, kind string, config *rest.Config) (bool, error) { + // Create a discovery client + discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) + if err != nil { + return false, fmt.Errorf("failed to create discovery client: %w", err) + } + + // Parse the apiVersion into a GroupVersion + _, err = schema.ParseGroupVersion(apiVersion) + if err != nil { + return false, fmt.Errorf("failed to parse apiVersion: %w", err) + } + + // Get the API Resources for the given GroupVersion + apiResources, err := discoveryClient.ServerResourcesForGroupVersion(apiVersion) + if err != nil { + if meta.IsNoMatchError(err) { + return false, nil // GroupVersion does not exist + } + return false, fmt.Errorf("failed to get server resources: %w", err) + } + + // Check if the kind exists in the API Resources + for _, resource := range apiResources.APIResources { + if resource.Kind == kind { + return true, nil + } + } + + return false, nil +} + +var rootObjectType = tracev1.ObjectType{ + APIVersion: kbappsv1.APIVersion, + Kind: kbappsv1.ClusterKind, +} + +var ( + defaultStateEvaluationExpression = tracev1.StateEvaluationExpression{ + CELExpression: &tracev1.CELExpression{ + Expression: "has(object.status.phase) && object.status.phase == \"Running\"", + }, + } + + defaultLocale = "en" + + eventGVK = schema.GroupVersionKind{ + Group: corev1.SchemeGroupVersion.Group, + Version: corev1.SchemeGroupVersion.Version, + Kind: constant.EventKind, + } +) + +// OwnershipRule defines an ownership rule between primary resource and its secondary resources. +type OwnershipRule struct { + // Primary specifies the primary object type. + // + Primary tracev1.ObjectType `json:"primary"` + + // OwnedResources specifies all the secondary resources of Primary. + // + OwnedResources []OwnedResource `json:"ownedResources"` +} + +// OwnedResource defines a secondary resource and the ownership criteria between its primary resource. +type OwnedResource struct { + // Secondary specifies the secondary object type. + // + Secondary tracev1.ObjectType `json:"secondary"` + + // Criteria specifies the ownership criteria with its primary resource. + // + Criteria OwnershipCriteria `json:"criteria"` +} + +// OwnershipCriteria defines an ownership criteria. +// Only one of SelectorCriteria, LabelCriteria or BuiltinRelationshipCriteria should be configured. +type OwnershipCriteria struct { + // SelectorCriteria specifies the selector field path in the primary object. + // For example, if the StatefulSet is the primary resource, selector will be "spec.selector". + // The selector field should be a map[string]string + // or LabelSelector (https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/label-selector/#LabelSelector) + // + // +optional + SelectorCriteria *FieldPath `json:"selectorCriteria,omitempty"` + + // LabelCriteria specifies the labels used to select the secondary objects. + // The value of each k-v pair can contain placeholder that will be replaced by the ReconciliationTrace Controller. + // Placeholder is formatted as "$(PLACEHOLDER)". + // Currently supported PLACEHOLDER: + // primary - same value as the primary object label with same key. + // primary.name - the name of the primary object. + // + // +optional + LabelCriteria map[string]string `json:"labelCriteria,omitempty"` + + // SpecifiedNameCriteria specifies the field from which to retrieve the secondary object name. + // + // +optional + SpecifiedNameCriteria *FieldPath `json:"specifiedNameCriteria,omitempty"` + + // Validation specifies the method to validate the OwnerReference of secondary resources. + // + // +kubebuilder:validation:Enum={Controller, Owner, None} + // +kubebuilder:default=Controller + // +optional + Validation ValidationType `json:"validation,omitempty"` +} + +// FieldPath defines a field path. +type FieldPath struct { + // Path of the field. + // + Path string `json:"path"` +} + +// ValidationType specifies the method to validate the OwnerReference of secondary resources. +type ValidationType string + +const ( + // ControllerValidation requires the secondary resource to have the primary resource + // in its OwnerReference with controller set to true. + ControllerValidation ValidationType = "Controller" + + // OwnerValidation requires the secondary resource to have the primary resource + // in its OwnerReference. + OwnerValidation ValidationType = "Owner" + + // NoValidation means no validation is performed on the OwnerReference. + NoValidation ValidationType = "None" +) + +type matchOwner struct { + controller bool + ownerUID types.UID +} + +type queryOptions struct { + matchLabels client.MatchingLabels + matchFields client.MatchingFields + matchOwner *matchOwner +} + +func (q *queryOptions) match(o client.Object) bool { + listOptions := &client.ListOptions{} + if q.matchLabels != nil { + q.matchLabels.ApplyToList(listOptions) + } + if q.matchFields != nil { + q.matchFields.ApplyToList(listOptions) + } + // default match + if listOptions.LabelSelector == nil && listOptions.FieldSelector == nil && q.matchOwner == nil { + return true + } + if listOptions.LabelSelector != nil && !listOptions.LabelSelector.Matches(labels.Set(o.GetLabels())) { + return false + } + if listOptions.FieldSelector != nil && + !listOptions.FieldSelector.Matches(fields.Set{"metadata.name": o.GetName()}) { + return false + } + if q.matchOwner != nil && !matchOwnerOf(q.matchOwner, o) { + return false + } + return true +} diff --git a/controllers/trace/util.go b/controllers/trace/util.go new file mode 100644 index 00000000000..d225249c736 --- /dev/null +++ b/controllers/trace/util.go @@ -0,0 +1,822 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "container/list" + "context" + "encoding/json" + "fmt" + "reflect" + "sort" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/controller/model" +) + +func objectTypeToGVK(objectType *tracev1.ObjectType) (*schema.GroupVersionKind, error) { + if objectType == nil { + return nil, nil + } + gv, err := schema.ParseGroupVersion(objectType.APIVersion) + if err != nil { + return nil, err + } + gvk := gv.WithKind(objectType.Kind) + return &gvk, nil +} + +func objectReferenceToType(objectRef *corev1.ObjectReference) *tracev1.ObjectType { + if objectRef == nil { + return nil + } + return &tracev1.ObjectType{ + APIVersion: objectRef.APIVersion, + Kind: objectRef.Kind, + } +} + +func objectReferenceToRef(reference *corev1.ObjectReference) *model.GVKNObjKey { + if reference == nil { + return nil + } + return &model.GVKNObjKey{ + GroupVersionKind: reference.GroupVersionKind(), + ObjectKey: client.ObjectKey{ + Namespace: reference.Namespace, + Name: reference.Name, + }, + } +} + +func objectRefToReference(objectRef model.GVKNObjKey, uid types.UID, resourceVersion string) *corev1.ObjectReference { + return &corev1.ObjectReference{ + APIVersion: objectRef.GroupVersionKind.GroupVersion().String(), + Kind: objectRef.Kind, + Namespace: objectRef.Namespace, + Name: objectRef.Name, + UID: uid, + ResourceVersion: resourceVersion, + } +} + +func objectRefToType(objectRef *model.GVKNObjKey) *tracev1.ObjectType { + if objectRef == nil { + return nil + } + return &tracev1.ObjectType{ + APIVersion: objectRef.GroupVersionKind.GroupVersion().String(), + Kind: objectRef.Kind, + } +} + +func objectType(apiVersion, kind string) tracev1.ObjectType { + return tracev1.ObjectType{ + APIVersion: apiVersion, + Kind: kind, + } +} + +func getObjectRef(object client.Object, scheme *runtime.Scheme) (*model.GVKNObjKey, error) { + gvk, err := apiutil.GVKForObject(object, scheme) + if err != nil { + return nil, err + } + return &model.GVKNObjKey{ + GroupVersionKind: gvk, + ObjectKey: client.ObjectKeyFromObject(object), + }, nil +} + +// getObjectReference creates a corev1.ObjectReference from a client.Object +func getObjectReference(obj client.Object, scheme *runtime.Scheme) (*corev1.ObjectReference, error) { + gvk, err := apiutil.GVKForObject(obj, scheme) + if err != nil { + return nil, err + } + + return &corev1.ObjectReference{ + APIVersion: gvk.GroupVersion().String(), + Kind: gvk.Kind, + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + UID: obj.GetUID(), + ResourceVersion: obj.GetResourceVersion(), + }, nil +} + +func getObjectReferenceString(n *tracev1.ObjectTreeNode) string { + if n == nil { + return "nil" + } + return strings.Join([]string{ + n.Primary.Kind, + n.Primary.Namespace, + n.Primary.Name, + n.Primary.APIVersion, + }, "-") +} + +func matchOwnerOf(owner *matchOwner, o client.Object) bool { + for _, ref := range o.GetOwnerReferences() { + if ref.UID == owner.ownerUID { + if !owner.controller { + return true + } + return ref.Controller != nil && *ref.Controller + } + } + return false +} + +func parseRevision(revisionStr string) int64 { + revision, err := strconv.ParseInt(revisionStr, 10, 64) + if err != nil { + revision = 0 + } + return revision +} + +// getObjectsByGVK gets all objects of a specific GVK. +// why not merge matchingFields into opts: +// fields.Selector needs the underlying cache to build an Indexer on the specific field, which is too heavy. +func getObjectsByGVK(ctx context.Context, cli client.Client, gvk *schema.GroupVersionKind, opts *queryOptions) ([]client.Object, error) { + runtimeObjectList, err := cli.Scheme().New(schema.GroupVersionKind{ + Group: gvk.Group, + Version: gvk.Version, + Kind: gvk.Kind + "List", + }) + if err != nil { + return nil, err + } + objectList, ok := runtimeObjectList.(client.ObjectList) + if !ok { + return nil, fmt.Errorf("list object is not a client.ObjectList for GVK %s", gvk) + } + var ml client.MatchingLabels + if opts != nil && opts.matchLabels != nil { + ml = opts.matchLabels + } + if err = cli.List(ctx, objectList, ml); err != nil { + return nil, err + } + runtimeObjects, err := meta.ExtractList(objectList) + if err != nil { + return nil, err + } + var objects []client.Object + listOptions := &client.ListOptions{} + if opts != nil && opts.matchFields != nil { + opts.matchFields.ApplyToList(listOptions) + } + for _, object := range runtimeObjects { + o, ok := object.(client.Object) + if !ok { + return nil, fmt.Errorf("object is not a client.Object for GVK %s", gvk) + } + if listOptions.FieldSelector != nil && !listOptions.FieldSelector.Matches(fields.Set{"metadata.name": o.GetName()}) { + continue + } + if opts != nil && opts.matchOwner != nil && !matchOwnerOf(opts.matchOwner, o) { + continue + } + objects = append(objects, o) + } + + return objects, nil +} + +func getObjectTreeFromCache(ctx context.Context, cli client.Client, primary client.Object, ownershipRules []OwnershipRule) (*tracev1.ObjectTreeNode, error) { + if primary == nil { + return nil, nil + } + + // primary tree node + reference, err := getObjectReference(primary, cli.Scheme()) + if err != nil { + return nil, err + } + tree := &tracev1.ObjectTreeNode{ + Primary: *reference, + } + + // secondary tree nodes + // find matched rules + primaryGVK, err := apiutil.GVKForObject(primary, cli.Scheme()) + if err != nil { + return nil, err + } + var matchedRules []OwnershipRule + for i := range ownershipRules { + rule := ownershipRules[i] + gvk, err := objectTypeToGVK(&rule.Primary) + if err != nil { + return nil, err + } + if *gvk == primaryGVK { + matchedRules = append(matchedRules, rule) + } + } + // build subtree + secondaries, err := getSecondaryObjects(ctx, cli, primary, matchedRules) + if err != nil { + return nil, err + } + for _, secondary := range secondaries { + subTree, err := getObjectTreeFromCache(ctx, cli, secondary, ownershipRules) + if err != nil { + return nil, err + } + tree.Secondaries = append(tree.Secondaries, subTree) + sort.SliceStable(tree.Secondaries, func(i, j int) bool { + return getObjectReferenceString(tree.Secondaries[i]) < getObjectReferenceString(tree.Secondaries[j]) + }) + } + + return tree, nil +} + +func getObjectsFromCache(ctx context.Context, cli client.Client, root *kbappsv1.Cluster, ownershipRules []OwnershipRule) (map[model.GVKNObjKey]client.Object, error) { + objectMap := make(map[model.GVKNObjKey]client.Object) + waitingList := list.New() + waitingList.PushFront(root) + for waitingList.Len() > 0 { + e := waitingList.Front() + waitingList.Remove(e) + obj, _ := e.Value.(client.Object) + objKey, err := getObjectRef(obj, cli.Scheme()) + if err != nil { + return nil, err + } + objectMap[*objKey] = obj + + secondaries, err := getSecondaryObjects(ctx, cli, obj, ownershipRules) + if err != nil { + return nil, err + } + for _, secondary := range secondaries { + waitingList.PushBack(secondary) + } + } + return objectMap, nil +} + +func getSecondaryObjects(ctx context.Context, cli client.Client, obj client.Object, ownershipRules []OwnershipRule) ([]client.Object, error) { + // find matched rules + rules, err := findMatchedRules(obj, ownershipRules, cli.Scheme()) + if err != nil { + return nil, err + } + + // get secondary objects + var secondaries []client.Object + for _, rule := range rules { + for _, ownedResource := range rule.OwnedResources { + gvk, err := objectTypeToGVK(&ownedResource.Secondary) + if err != nil { + return nil, err + } + opts, err := parseQueryOptions(obj, &ownedResource.Criteria) + if err != nil { + return nil, err + } + objects, err := getObjectsByGVK(ctx, cli, gvk, opts) + if err != nil { + return nil, err + } + secondaries = append(secondaries, objects...) + } + } + + return secondaries, nil +} + +func parseQueryOptions(primary client.Object, criteria *OwnershipCriteria) (*queryOptions, error) { + opts := &queryOptions{} + if criteria.SelectorCriteria != nil { + ml, err := parseSelector(primary, criteria.SelectorCriteria.Path) + if err != nil { + return nil, err + } + opts.matchLabels = ml + } + if criteria.LabelCriteria != nil { + labels := make(map[string]string, len(criteria.LabelCriteria)) + for k, v := range criteria.LabelCriteria { + value := strings.ReplaceAll(v, "$(primary.name)", primary.GetName()) + value = strings.ReplaceAll(value, "$(primary)", primary.GetLabels()[k]) + labels[k] = value + } + opts.matchLabels = labels + } + if criteria.SpecifiedNameCriteria != nil { + fieldMap, err := flattenObject(primary) + if err != nil { + return nil, err + } + name, ok := fieldMap[criteria.SpecifiedNameCriteria.Path] + if ok { + opts.matchFields = client.MatchingFields{"metadata.name": name} + } + } + if criteria.Validation != "" && criteria.Validation != NoValidation { + opts.matchOwner = &matchOwner{ + ownerUID: primary.GetUID(), + controller: criteria.Validation == ControllerValidation, + } + } + return opts, nil +} + +// parseSelector checks if a field exists in the object and returns it if it's a metav1.LabelSelector +func parseSelector(obj client.Object, fieldPath string) (map[string]string, error) { + selectorField, err := parseField(obj, fieldPath) + if err != nil { + return nil, err + } + // Attempt to convert the final field to a LabelSelector + // TODO(free6om): handle metav1.LabelSelector + // labelSelector := &metav1.LabelSelector{} + labelSelector := make(map[string]string) + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(selectorField, &labelSelector); err != nil { + return nil, fmt.Errorf("failed to parse as LabelSelector: %w", err) + } + + return labelSelector, nil +} + +func parseField(obj client.Object, fieldPath string) (map[string]interface{}, error) { + unstructuredObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + return nil, fmt.Errorf("failed to convert to unstructured: %w", err) + } + + // Use the field path to find the field + pathParts := strings.Split(fieldPath, ".") + current := unstructuredObj + for i := 0; i < len(pathParts); i++ { + part := pathParts[i] + if next, ok := current[part].(map[string]interface{}); ok { + current = next + } else { + return nil, fmt.Errorf("field '%s' does not exist", fieldPath) + } + } + return current, nil +} + +func specMapToJSON(spec interface{}) []byte { + // Convert the spec map to JSON for the patch functions + specJSON, _ := json.Marshal(spec) + return specJSON +} + +// convertToMap converts a client.Object to a map respecting JSON tags. +func convertToMap(obj client.Object) (map[string]interface{}, error) { + objBytes, err := json.Marshal(obj) + if err != nil { + return nil, err + } + var objMap map[string]interface{} + if err := json.Unmarshal(objBytes, &objMap); err != nil { + return nil, err + } + return objMap, nil +} + +// flattenJSON flattens a nested JSON object into a single-level map. +func flattenJSON(data map[string]interface{}, prefix string, flatMap map[string]string) { + for key, value := range data { + newKey := key + if prefix != "" { + newKey = prefix + "." + key + } + + switch v := value.(type) { + case map[string]interface{}: + flattenJSON(v, newKey, flatMap) + case []interface{}: + for i, item := range v { + flattenJSON(map[string]interface{}{fmt.Sprintf("%d", i): item}, newKey, flatMap) + } + default: + flatMap[newKey] = fmt.Sprintf("%v", v) + } + } +} + +// flattenObject converts a client.Object to a flattened map. +func flattenObject(obj client.Object) (map[string]string, error) { + objMap, err := convertToMap(obj) + if err != nil { + return nil, err + } + + flatMap := make(map[string]string) + flattenJSON(objMap, "", flatMap) + return flatMap, nil +} + +func buildObjectSummaries(initialObjectMap, newObjectMap map[model.GVKNObjKey]client.Object) []tracev1.ObjectSummary { + initialObjectSet, newObjectSet := sets.KeySet(initialObjectMap), sets.KeySet(newObjectMap) + createSet := newObjectSet.Difference(initialObjectSet) + updateSet := newObjectSet.Intersection(initialObjectSet) + deleteSet := initialObjectSet.Difference(newObjectSet) + summaryMap := make(map[tracev1.ObjectType]*tracev1.ObjectSummary) + doCount := func(s sets.Set[model.GVKNObjKey], summaryUpdater func(objectRef *model.GVKNObjKey, summary *tracev1.ObjectSummary)) { + for objectRef := range s { + key := *objectRefToType(&objectRef) + summary, ok := summaryMap[key] + if !ok { + summary = &tracev1.ObjectSummary{ + ObjectType: key, + Total: 0, + } + summaryMap[key] = summary + } + if summary.ChangeSummary == nil { + summary.ChangeSummary = &tracev1.ObjectChangeSummary{} + } + summaryUpdater(&objectRef, summary) + } + } + doCount(createSet, func(_ *model.GVKNObjKey, summary *tracev1.ObjectSummary) { + summary.Total += 1 + if summary.ChangeSummary.Added == nil { + summary.ChangeSummary.Added = pointer.Int32(0) + } + *summary.ChangeSummary.Added += 1 + }) + doCount(updateSet, func(objectRef *model.GVKNObjKey, summary *tracev1.ObjectSummary) { + initialObj := initialObjectMap[*objectRef] + newObj := newObjectMap[*objectRef] + summary.Total += 1 + if initialObj != nil && newObj != nil && initialObj.GetResourceVersion() == newObj.GetResourceVersion() { + return + } + if summary.ChangeSummary.Updated == nil { + summary.ChangeSummary.Updated = pointer.Int32(0) + } + *summary.ChangeSummary.Updated += 1 + }) + doCount(deleteSet, func(_ *model.GVKNObjKey, summary *tracev1.ObjectSummary) { + if summary.ChangeSummary.Deleted == nil { + summary.ChangeSummary.Deleted = pointer.Int32(0) + } + *summary.ChangeSummary.Deleted += 1 + }) + + var objectSummaries []tracev1.ObjectSummary + for _, summary := range summaryMap { + objectSummaries = append(objectSummaries, *summary) + } + sort.SliceStable(objectSummaries, func(i, j int) bool { + a, b := &objectSummaries[i], &objectSummaries[j] + if a.ObjectType.APIVersion != b.ObjectType.APIVersion { + return a.ObjectType.APIVersion < b.ObjectType.APIVersion + } + return a.ObjectType.Kind < b.ObjectType.Kind + }) + + return objectSummaries +} + +func buildChanges(oldObjectMap, newObjectMap map[model.GVKNObjKey]client.Object, + descriptionFormat func(client.Object, client.Object, tracev1.ObjectChangeType, *schema.GroupVersionKind) (string, *string)) []tracev1.ObjectChange { + // calculate createSet, deleteSet and updateSet + newObjectSet := sets.KeySet(newObjectMap) + oldObjectSet := sets.KeySet(oldObjectMap) + createSet := newObjectSet.Difference(oldObjectSet) + updateSet := newObjectSet.Intersection(oldObjectSet) + deleteSet := oldObjectSet.Difference(newObjectSet) + + // build new slice of reconciliation changes from last round calculation + var changes []tracev1.ObjectChange + allObjectMap := map[tracev1.ObjectChangeType]sets.Set[model.GVKNObjKey]{ + tracev1.ObjectCreationType: createSet, + tracev1.ObjectUpdateType: updateSet, + tracev1.ObjectDeletionType: deleteSet, + } + // Why not use for-range on allObjectMap: + // Order is important here. A for-range on a map can't guarantee the built changes' order. + for _, changeType := range []tracev1.ObjectChangeType{tracev1.ObjectCreationType, tracev1.ObjectUpdateType, tracev1.ObjectDeletionType} { + changeSet := allObjectMap[changeType] + for key := range changeSet { + var oldObj, newObj client.Object + if oldObjectMap != nil { + oldObj = oldObjectMap[key] + } + if newObjectMap != nil { + newObj = newObjectMap[key] + } + obj := newObj + if changeType == tracev1.ObjectDeletionType { + obj = oldObj + } + if changeType == tracev1.ObjectUpdateType && + (oldObj == nil || newObj == nil || oldObj.GetResourceVersion() == newObj.GetResourceVersion()) { + continue + } + isEvent := isEvent(&key.GroupVersionKind) + if isEvent && changeType == tracev1.ObjectDeletionType { + continue + } + var ( + ref *corev1.ObjectReference + eventAttributes *tracev1.EventAttributes + ) + if isEvent { + changeType = tracev1.EventType + evt, _ := obj.(*corev1.Event) + ref = &evt.InvolvedObject + eventAttributes = &tracev1.EventAttributes{ + Name: evt.Name, + Type: evt.Type, + Reason: evt.Reason, + } + } else { + ref = objectRefToReference(key, obj.GetUID(), obj.GetResourceVersion()) + } + description, localDescription := descriptionFormat(oldObj, newObj, changeType, &key.GroupVersionKind) + change := tracev1.ObjectChange{ + ObjectReference: *ref, + ChangeType: changeType, + EventAttributes: eventAttributes, + Revision: parseRevision(obj.GetResourceVersion()), + Timestamp: func() *metav1.Time { t := metav1.Now(); return &t }(), + Description: description, + LocalDescription: localDescription, + } + changes = append(changes, change) + } + } + return changes +} + +func buildDescriptionFormatter(i18nResources *corev1.ConfigMap, defaultLocale string, locale *string) func(client.Object, client.Object, tracev1.ObjectChangeType, *schema.GroupVersionKind) (string, *string) { + return func(oldObj client.Object, newObj client.Object, changeType tracev1.ObjectChangeType, gvk *schema.GroupVersionKind) (string, *string) { + description := formatDescription(oldObj, newObj, changeType, gvk, i18nResources, &defaultLocale) + localDescription := formatDescription(oldObj, newObj, changeType, gvk, i18nResources, locale) + return *description, localDescription + } +} + +func formatDescription(oldObj, newObj client.Object, changeType tracev1.ObjectChangeType, gvk *schema.GroupVersionKind, i18nResource *corev1.ConfigMap, locale *string) *string { + if locale == nil { + return nil + } + defaultStr := pointer.String(string(changeType)) + if oldObj == nil && newObj == nil { + return defaultStr + } + if err := defaultResourcesManager.ParseRaw(i18nResource); err != nil { + return defaultStr + } + obj := newObj + if obj == nil { + obj = oldObj + } + var ( + key string + needFormat bool + ) + if isEvent(gvk) { + evt, _ := obj.(*corev1.Event) + defaultStr = pointer.String(evt.Message) + key = evt.Message + } else { + key = fmt.Sprintf("%s/%s/%s", gvk.GroupVersion().String(), gvk.Kind, changeType) + needFormat = true + } + formatString := defaultResourcesManager.GetFormatString(key, *locale) + if len(formatString) == 0 { + return defaultStr + } + result := formatString + if needFormat { + result = fmt.Sprintf(formatString, obj.GetNamespace(), obj.GetName()) + } + return pointer.String(result) +} + +func isEvent(gvk *schema.GroupVersionKind) bool { + return *gvk == eventGVK +} + +func getObjectsFromTree(tree *tracev1.ObjectTreeNode, store ObjectRevisionStore, scheme *runtime.Scheme) (map[model.GVKNObjKey]client.Object, error) { + if tree == nil { + return nil, nil + } + objectRef := objectReferenceToRef(&tree.Primary) + revision := parseRevision(tree.Primary.ResourceVersion) + obj, err := store.Get(objectRef, revision) + if err != nil && !apierrors.IsNotFound(err) { + return nil, err + } + objectMap := make(map[model.GVKNObjKey]client.Object) + // cache loss after controller restarted, mock one + if obj == nil { + ro, err := scheme.New(objectRef.GroupVersionKind) + if err != nil { + return nil, err + } + obj, _ = ro.(client.Object) + obj.SetNamespace(tree.Primary.Namespace) + obj.SetName(tree.Primary.Name) + obj.SetResourceVersion(tree.Primary.ResourceVersion) + obj.SetUID(tree.Primary.UID) + } + objectMap[*objectRef] = obj + + for _, treeNode := range tree.Secondaries { + secondaryMap, err := getObjectsFromTree(treeNode, store, scheme) + if err != nil { + return nil, err + } + for key, object := range secondaryMap { + objectMap[key] = object + } + } + return objectMap, nil +} + +func getObjectTreeWithRevision(primary client.Object, ownershipRules []OwnershipRule, store ObjectRevisionStore, revision int64, scheme *runtime.Scheme) (*tracev1.ObjectTreeNode, error) { + // find matched rules + matchedRules, err := findMatchedRules(primary, ownershipRules, scheme) + if err != nil { + return nil, err + } + + reference, err := getObjectReference(primary, scheme) + if err != nil { + return nil, err + } + tree := &tracev1.ObjectTreeNode{ + Primary: *reference, + } + // traverse rules, build subtree + var secondaries []client.Object + for _, rule := range matchedRules { + for _, ownedResource := range rule.OwnedResources { + gvk, err := objectTypeToGVK(&ownedResource.Secondary) + if err != nil { + return nil, err + } + objects := getObjectsByRevision(gvk, store, revision) + objects, err = filterByCriteria(primary, objects, &ownedResource.Criteria) + if err != nil { + return nil, err + } + secondaries = append(secondaries, objects...) + } + } + for _, secondary := range secondaries { + subTree, err := getObjectTreeWithRevision(secondary, ownershipRules, store, revision, scheme) + if err != nil { + return nil, err + } + tree.Secondaries = append(tree.Secondaries, subTree) + sort.SliceStable(tree.Secondaries, func(i, j int) bool { + return getObjectReferenceString(tree.Secondaries[i]) < getObjectReferenceString(tree.Secondaries[j]) + }) + } + + return tree, nil +} + +func findMatchedRules(obj client.Object, ownershipRules []OwnershipRule, scheme *runtime.Scheme) ([]*OwnershipRule, error) { + targetGVK, err := apiutil.GVKForObject(obj, scheme) + if err != nil { + return nil, err + } + var matchedRules []*OwnershipRule + for i := range ownershipRules { + rule := &ownershipRules[i] + gvk, err := objectTypeToGVK(&rule.Primary) + if err != nil { + return nil, err + } + + if *gvk == targetGVK { + matchedRules = append(matchedRules, rule) + } + } + return matchedRules, nil +} + +func filterByCriteria(primary client.Object, objects []client.Object, criteria *OwnershipCriteria) ([]client.Object, error) { + var matchedObjects []client.Object + opts, err := parseQueryOptions(primary, criteria) + if err != nil { + return nil, err + } + for _, object := range objects { + if opts.match(object) { + matchedObjects = append(matchedObjects, object) + } + } + return matchedObjects, nil +} + +func getObjectsByRevision(gvk *schema.GroupVersionKind, store ObjectRevisionStore, revision int64) []client.Object { + objectMap := store.List(gvk) + var matchedObjects []client.Object + for _, revisionMap := range objectMap { + rev := int64(-1) + for r := range revisionMap { + if rev < r && r <= revision { + rev = r + } + } + if rev > -1 { + matchedObjects = append(matchedObjects, revisionMap[rev]) + } + } + return matchedObjects +} + +func deleteUnusedRevisions(store ObjectRevisionStore, changes []tracev1.ObjectChange, reference client.Object) { + for _, change := range changes { + objectRef := objectReferenceToRef(&change.ObjectReference) + if change.ChangeType == tracev1.EventType { + objectRef.GroupVersionKind = eventGVK + objectRef.Name = change.EventAttributes.Name + } + store.Delete(objectRef, reference, change.Revision) + } +} + +// getFieldAsStruct extracts the field with name of fieldName from a client.Object and returns it as an interface{}. +func getFieldAsStruct(obj client.Object, fieldName string) (interface{}, error) { + // Get the value of the object + objValue := reflect.ValueOf(obj) + + // Check if the object is a pointer to a struct + if objValue.Kind() != reflect.Ptr || objValue.Elem().Kind() != reflect.Struct { + return nil, fmt.Errorf("obj must be a pointer to a struct") + } + + // Get the Spec field + specField := objValue.Elem().FieldByName(fieldName) + if !specField.IsValid() { + return nil, fmt.Errorf("spec field not found") + } + + // Return the Spec field as an interface{} + return specField.Interface(), nil +} + +// normalize normalizes the default value of fields to a uniform format, for example, a list with a length of 0 is normalized to nil. +// The purpose is making the object easily to be compared by reflect.DeepEqual. +func normalize(obj client.Object) (client.Object, error) { + if obj == nil { + return nil, nil + } + data, err := json.Marshal(obj) + if err != nil { + return nil, err + } + t := reflect.TypeOf(obj) + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + newObj := reflect.New(t).Interface().(client.Object) + err = json.Unmarshal(data, newObj) + if err != nil { + return nil, err + } + return newObj, nil +} diff --git a/controllers/trace/util_test.go b/controllers/trace/util_test.go new file mode 100644 index 00000000000..3e22653e83d --- /dev/null +++ b/controllers/trace/util_test.go @@ -0,0 +1,628 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package trace + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/golang/mock/gomock" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + + kbappsv1 "github.com/apecloud/kubeblocks/apis/apps/v1" + tracev1 "github.com/apecloud/kubeblocks/apis/trace/v1" + "github.com/apecloud/kubeblocks/pkg/constant" + "github.com/apecloud/kubeblocks/pkg/controller/builder" + "github.com/apecloud/kubeblocks/pkg/controller/model" + testutil "github.com/apecloud/kubeblocks/pkg/testutil/k8s" + "github.com/apecloud/kubeblocks/pkg/testutil/k8s/mocks" +) + +var _ = Describe("util test", func() { + var ( + k8sMock *mocks.MockClient + controller *gomock.Controller + ) + + BeforeEach(func() { + controller, k8sMock = testutil.SetupK8sMock() + }) + + AfterEach(func() { + controller.Finish() + }) + + Context("objectTypeToGVK", func() { + It("should work well", func() { + objType := &tracev1.ObjectType{ + APIVersion: tracev1.SchemeBuilder.GroupVersion.String(), + Kind: tracev1.Kind, + } + gvk, err := objectTypeToGVK(objType) + Expect(err).Should(BeNil()) + Expect(*gvk).Should(Equal(schema.GroupVersionKind{ + Group: tracev1.GroupVersion.Group, + Version: tracev1.GroupVersion.Version, + Kind: tracev1.Kind, + })) + + objType = nil + gvk, err = objectTypeToGVK(objType) + Expect(err).Should(BeNil()) + Expect(gvk).Should(BeNil()) + }) + }) + + Context("objectReferenceToType", func() { + It("should work well", func() { + ref := &corev1.ObjectReference{ + APIVersion: tracev1.SchemeBuilder.GroupVersion.String(), + Kind: tracev1.Kind, + } + objType := objectReferenceToType(ref) + Expect(objType).ShouldNot(BeNil()) + Expect(*objType).Should(Equal(tracev1.ObjectType{ + APIVersion: tracev1.SchemeBuilder.GroupVersion.String(), + Kind: tracev1.Kind, + })) + + ref = nil + objType = objectReferenceToType(ref) + Expect(objType).Should(BeNil()) + }) + }) + + Context("objectReferenceToRef", func() { + It("should work well", func() { + ref := &corev1.ObjectReference{ + APIVersion: tracev1.SchemeBuilder.GroupVersion.String(), + Kind: tracev1.Kind, + Namespace: "foo", + Name: "bar", + } + objRef := objectReferenceToRef(ref) + Expect(objRef).ShouldNot(BeNil()) + Expect(*objRef).Should(Equal(model.GVKNObjKey{ + GroupVersionKind: schema.GroupVersionKind{ + Group: tracev1.GroupVersion.Group, + Version: tracev1.GroupVersion.Version, + Kind: tracev1.Kind, + }, + ObjectKey: client.ObjectKey{ + Namespace: "foo", + Name: "bar", + }, + })) + + ref = nil + objRef = objectReferenceToRef(ref) + Expect(objRef).Should(BeNil()) + }) + }) + + Context("objectRefToReference", func() { + It("should work well", func() { + objectRef := model.GVKNObjKey{ + GroupVersionKind: schema.GroupVersionKind{ + Group: tracev1.GroupVersion.Group, + Version: tracev1.GroupVersion.Version, + Kind: tracev1.Kind, + }, + ObjectKey: client.ObjectKey{ + Namespace: "foo", + Name: "bar", + }, + } + uid := uuid.NewUUID() + resourceVersion := "123456" + ref := objectRefToReference(objectRef, uid, resourceVersion) + Expect(ref).ShouldNot(BeNil()) + Expect(*ref).Should(Equal(corev1.ObjectReference{ + APIVersion: tracev1.SchemeBuilder.GroupVersion.String(), + Kind: tracev1.Kind, + Namespace: "foo", + Name: "bar", + UID: uid, + ResourceVersion: resourceVersion, + })) + }) + }) + + Context("objectRefToType", func() { + It("should work well", func() { + objectRef := &model.GVKNObjKey{ + GroupVersionKind: schema.GroupVersionKind{ + Group: tracev1.GroupVersion.Group, + Version: tracev1.GroupVersion.Version, + Kind: tracev1.Kind, + }, + ObjectKey: client.ObjectKey{ + Namespace: "foo", + Name: "bar", + }, + } + t := objectRefToType(objectRef) + Expect(t).ShouldNot(BeNil()) + Expect(*t).Should(Equal(tracev1.ObjectType{ + APIVersion: tracev1.SchemeBuilder.GroupVersion.String(), + Kind: tracev1.Kind, + })) + objectRef = nil + t = objectRefToType(objectRef) + Expect(t).Should(BeNil()) + }) + }) + + Context("getObjectRef", func() { + It("should work well", func() { + obj := builder.NewClusterBuilder(namespace, name).GetObject() + objectRef, err := getObjectRef(obj, scheme.Scheme) + Expect(err).Should(BeNil()) + Expect(*objectRef).Should(Equal(model.GVKNObjKey{ + GroupVersionKind: schema.GroupVersionKind{ + Group: kbappsv1.GroupVersion.Group, + Version: kbappsv1.GroupVersion.Version, + Kind: kbappsv1.ClusterKind, + }, + ObjectKey: client.ObjectKey{ + Namespace: namespace, + Name: name, + }, + })) + }) + }) + + Context("getObjectReference", func() { + It("should work well", func() { + obj := builder.NewClusterBuilder(namespace, name).SetUID(uid).SetResourceVersion(resourceVersion).GetObject() + ref, err := getObjectReference(obj, scheme.Scheme) + Expect(err).Should(BeNil()) + Expect(*ref).Should(Equal(corev1.ObjectReference{ + APIVersion: kbappsv1.APIVersion, + Kind: kbappsv1.ClusterKind, + Namespace: namespace, + Name: name, + UID: uid, + ResourceVersion: resourceVersion, + })) + }) + }) + + Context("matchOwnerOf", func() { + It("should work well", func() { + owner := builder.NewClusterBuilder(namespace, name).SetUID(uid).GetObject() + compName := "hello" + fullCompName := fmt.Sprintf("%s-%s", owner.Name, compName) + owned := builder.NewComponentBuilder(namespace, fullCompName, ""). + SetOwnerReferences(kbappsv1.APIVersion, kbappsv1.ClusterKind, owner). + GetObject() + matchOwner := &matchOwner{ + controller: true, + ownerUID: uid, + } + Expect(matchOwnerOf(matchOwner, owned)).Should(BeTrue()) + }) + }) + + Context("parseRevision", func() { + It("should work well", func() { + rev := parseRevision(resourceVersion) + Expect(rev).Should(Equal(int64(612345))) + }) + }) + + Context("parseQueryOptions", func() { + It("should work well", func() { + By("selector criteria") + labels := map[string]string{ + "label1": "value1", + "label2": "value2", + } + primary := builder.NewInstanceSetBuilder(namespace, name).AddLabelsInMap(labels).AddMatchLabelsInMap(labels).GetObject() + criteria := &OwnershipCriteria{ + SelectorCriteria: &FieldPath{ + Path: "spec.selector.matchLabels", + }, + } + opt, err := parseQueryOptions(primary, criteria) + Expect(err).Should(BeNil()) + Expect(opt).ShouldNot(BeNil()) + Expect(opt.matchLabels).Should(BeEquivalentTo(labels)) + + By("label criteria") + labels = map[string]string{ + "label1": "$(primary)", + "label2": "$(primary.name)", + "label3": "value3", + } + criteria = &OwnershipCriteria{ + LabelCriteria: labels, + } + opt, err = parseQueryOptions(primary, criteria) + Expect(err).Should(BeNil()) + Expect(opt).ShouldNot(BeNil()) + expectedLabels := map[string]string{ + "label1": primary.Labels["label1"], + "label2": primary.Name, + "label3": "value3", + } + Expect(opt.matchLabels).Should(BeEquivalentTo(expectedLabels)) + + By("specified name criteria") + criteria = &OwnershipCriteria{ + SpecifiedNameCriteria: &FieldPath{ + Path: "metadata.name", + }, + } + opt, err = parseQueryOptions(primary, criteria) + Expect(err).Should(BeNil()) + Expect(opt).ShouldNot(BeNil()) + Expect(opt.matchFields).Should(BeEquivalentTo(map[string]string{"metadata.name": primary.Name})) + + By("validation type") + criteria = &OwnershipCriteria{ + Validation: ControllerValidation, + } + opt, err = parseQueryOptions(primary, criteria) + Expect(err).Should(BeNil()) + Expect(opt).ShouldNot(BeNil()) + Expect(opt.matchOwner).ShouldNot(BeNil()) + Expect(*opt.matchOwner).Should(Equal(matchOwner{ownerUID: primary.UID, controller: true})) + }) + }) + + Context("getObjectsByGVK", func() { + It("should work well", func() { + gvk := &schema.GroupVersionKind{ + Group: kbappsv1.GroupVersion.Group, + Version: kbappsv1.GroupVersion.Version, + Kind: kbappsv1.ComponentKind, + } + opt := &queryOptions{ + matchOwner: &matchOwner{ + controller: true, + ownerUID: uid, + }, + } + owner := builder.NewClusterBuilder(namespace, name).SetUID(uid).GetObject() + compName := "hello" + fullCompName := fmt.Sprintf("%s-%s", owner.Name, compName) + owned := builder.NewComponentBuilder(namespace, fullCompName, ""). + SetOwnerReferences(kbappsv1.APIVersion, kbappsv1.ClusterKind, owner). + GetObject() + k8sMock.EXPECT().Scheme().Return(scheme.Scheme).AnyTimes() + k8sMock.EXPECT(). + List(gomock.Any(), &kbappsv1.ComponentList{}, gomock.Any()). + DoAndReturn(func(_ context.Context, list *kbappsv1.ComponentList, _ ...client.ListOption) error { + list.Items = []kbappsv1.Component{*owned} + return nil + }).Times(1) + + objects, err := getObjectsByGVK(ctx, k8sMock, gvk, opt) + Expect(err).Should(BeNil()) + Expect(objects).Should(HaveLen(1)) + Expect(objects[0]).Should(Equal(owned)) + }) + }) + + Context("get objects from cache", func() { + var ( + primary *kbappsv1.Cluster + secondaries []kbappsv1.Component + ) + BeforeEach(func() { + primary, secondaries = mockObjects(k8sMock) + }) + + Context("getObjectTreeFromCache", func() { + It("should work well", func() { + tree, err := getObjectTreeFromCache(ctx, k8sMock, primary, getKBOwnershipRules()) + Expect(err).Should(BeNil()) + Expect(tree).ShouldNot(BeNil()) + Expect(tree.Primary).Should(Equal(corev1.ObjectReference{ + APIVersion: kbappsv1.SchemeBuilder.GroupVersion.String(), + Kind: kbappsv1.ClusterKind, + Namespace: primary.Namespace, + Name: primary.Name, + UID: primary.UID, + ResourceVersion: primary.ResourceVersion, + })) + Expect(tree.Secondaries).Should(HaveLen(2)) + for i := 0; i < len(secondaries); i++ { + Expect(tree.Secondaries[i].Primary).Should(Equal(corev1.ObjectReference{ + APIVersion: kbappsv1.SchemeBuilder.GroupVersion.String(), + Kind: kbappsv1.ComponentKind, + Namespace: secondaries[i].Namespace, + Name: secondaries[i].Name, + UID: secondaries[i].UID, + ResourceVersion: secondaries[i].ResourceVersion, + })) + } + }) + }) + + Context("getObjectsFromCache", func() { + It("should work well", func() { + objects, err := getObjectsFromCache(ctx, k8sMock, primary, getKBOwnershipRules()) + Expect(err).Should(BeNil()) + Expect(objects).Should(HaveLen(3)) + expectedObjects := make(map[model.GVKNObjKey]client.Object, len(objects)) + for _, object := range []client.Object{primary, &secondaries[0], &secondaries[1]} { + objectRef, err := getObjectRef(object, k8sMock.Scheme()) + Expect(err).Should(BeNil()) + expectedObjects[*objectRef] = object + } + for key, object := range expectedObjects { + v, ok := objects[key] + Expect(ok).Should(BeTrue()) + Expect(v).Should(Equal(object)) + } + }) + }) + }) + + Context("Changes And Summary", func() { + var initialObjectMap, newObjectMap map[model.GVKNObjKey]client.Object + var objectList []client.Object + + BeforeEach(func() { + initialObjectList := []client.Object{ + builder.NewComponentBuilder(namespace, name+"-0", "").GetObject(), + builder.NewComponentBuilder(namespace, name+"-1", "").GetObject(), + } + newObjectList := []client.Object{ + builder.NewComponentBuilder(namespace, name+"-0", "").GetObject(), + builder.NewComponentBuilder(namespace, name+"-2", "").GetObject(), + } + newObjectList[0].SetResourceVersion(resourceVersion) + objectList = []client.Object{newObjectList[1], newObjectList[0], initialObjectList[1]} + + initialObjectMap = make(map[model.GVKNObjKey]client.Object, len(initialObjectList)) + newObjectMap = make(map[model.GVKNObjKey]client.Object, len(newObjectList)) + for _, object := range initialObjectList { + objectRef, err := getObjectRef(object, scheme.Scheme) + Expect(err).Should(BeNil()) + initialObjectMap[*objectRef] = object + } + for _, object := range newObjectList { + objectRef, err := getObjectRef(object, scheme.Scheme) + Expect(err).Should(BeNil()) + newObjectMap[*objectRef] = object + } + }) + + Context("buildObjectSummaries", func() { + It("should work well", func() { + summary := buildObjectSummaries(initialObjectMap, newObjectMap) + Expect(summary).Should(HaveLen(1)) + Expect(summary[0].ObjectType).Should(Equal(tracev1.ObjectType{ + APIVersion: kbappsv1.SchemeBuilder.GroupVersion.String(), + Kind: kbappsv1.ComponentKind, + })) + Expect(summary[0].Total).Should(BeEquivalentTo(2)) + Expect(summary[0].ChangeSummary).ShouldNot(BeNil()) + Expect(summary[0].ChangeSummary.Added).ShouldNot(BeNil()) + Expect(*summary[0].ChangeSummary.Added).Should(BeEquivalentTo(1)) + Expect(summary[0].ChangeSummary.Updated).ShouldNot(BeNil()) + Expect(*summary[0].ChangeSummary.Updated).Should(BeEquivalentTo(1)) + Expect(summary[0].ChangeSummary.Deleted).ShouldNot(BeNil()) + Expect(*summary[0].ChangeSummary.Deleted).Should(BeEquivalentTo(1)) + }) + }) + + Context("buildChanges", func() { + It("should work well", func() { + i18n := builder.NewConfigMapBuilder(namespace, name).SetData( + map[string]string{"en": "apps.kubeblocks.io/v1/Component/Creation=Component %s/%s is created."}, + ).GetObject() + changes := buildChanges(initialObjectMap, newObjectMap, buildDescriptionFormatter(i18n, defaultLocale, nil)) + Expect(changes).Should(HaveLen(3)) + Expect(changes[0]).Should(Equal(tracev1.ObjectChange{ + ObjectReference: corev1.ObjectReference{ + APIVersion: kbappsv1.SchemeBuilder.GroupVersion.String(), + Kind: kbappsv1.ComponentKind, + Namespace: objectList[0].GetNamespace(), + Name: objectList[0].GetName(), + UID: objectList[0].GetUID(), + ResourceVersion: objectList[0].GetResourceVersion(), + }, + ChangeType: tracev1.ObjectCreationType, + Revision: parseRevision(objectList[0].GetResourceVersion()), + Timestamp: changes[0].Timestamp, + Description: fmt.Sprintf("Component %s/%s is created.", objectList[0].GetNamespace(), objectList[0].GetName()), + })) + Expect(changes[1]).Should(Equal(tracev1.ObjectChange{ + ObjectReference: corev1.ObjectReference{ + APIVersion: kbappsv1.SchemeBuilder.GroupVersion.String(), + Kind: kbappsv1.ComponentKind, + Namespace: objectList[1].GetNamespace(), + Name: objectList[1].GetName(), + UID: objectList[1].GetUID(), + ResourceVersion: objectList[1].GetResourceVersion(), + }, + ChangeType: tracev1.ObjectUpdateType, + Revision: parseRevision(objectList[1].GetResourceVersion()), + Timestamp: changes[1].Timestamp, + Description: "Update", + })) + Expect(changes[2]).Should(Equal(tracev1.ObjectChange{ + ObjectReference: corev1.ObjectReference{ + APIVersion: kbappsv1.SchemeBuilder.GroupVersion.String(), + Kind: kbappsv1.ComponentKind, + Namespace: objectList[2].GetNamespace(), + Name: objectList[2].GetName(), + UID: objectList[2].GetUID(), + ResourceVersion: objectList[2].GetResourceVersion(), + }, + ChangeType: tracev1.ObjectDeletionType, + Revision: parseRevision(objectList[2].GetResourceVersion()), + Timestamp: changes[2].Timestamp, + Description: "Deletion", + })) + }) + }) + }) + + Context("get objects from store", func() { + var ( + primary *kbappsv1.Cluster + secondaries []kbappsv1.Component + store ObjectRevisionStore + reference client.Object + ) + BeforeEach(func() { + primary = builder.NewClusterBuilder(namespace, name).SetUID(uid).SetResourceVersion(resourceVersion).GetObject() + compNames := []string{"hello", "world"} + secondaries = nil + for _, compName := range compNames { + fullCompName := fmt.Sprintf("%s-%s", primary.Name, compName) + secondary := builder.NewComponentBuilder(namespace, fullCompName, ""). + SetOwnerReferences(kbappsv1.APIVersion, kbappsv1.ClusterKind, primary). + AddLabels(constant.AppManagedByLabelKey, constant.AppName). + AddLabels(constant.AppInstanceLabelKey, primary.Name). + SetUID(uid). + GetObject() + secondary.ResourceVersion = resourceVersion + secondaries = append(secondaries, *secondary) + } + + store = NewObjectStore(scheme.Scheme) + reference = &tracev1.ReconciliationTrace{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + } + Expect(store.Insert(primary, reference)).Should(Succeed()) + Expect(store.Insert(&secondaries[0], reference)).Should(Succeed()) + Expect(store.Insert(&secondaries[1], reference)).Should(Succeed()) + }) + + Context("getObjectsFromTree", func() { + It("should work well", func() { + tree := &tracev1.ObjectTreeNode{ + Primary: corev1.ObjectReference{ + APIVersion: kbappsv1.SchemeBuilder.GroupVersion.String(), + Kind: kbappsv1.ClusterKind, + Namespace: primary.Namespace, + Name: primary.Name, + UID: primary.UID, + ResourceVersion: primary.ResourceVersion, + }, + Secondaries: []*tracev1.ObjectTreeNode{ + { + Primary: corev1.ObjectReference{ + APIVersion: kbappsv1.SchemeBuilder.GroupVersion.String(), + Kind: kbappsv1.ComponentKind, + Namespace: secondaries[0].Namespace, + Name: secondaries[0].Name, + UID: secondaries[0].UID, + ResourceVersion: secondaries[0].ResourceVersion, + }, + }, + { + Primary: corev1.ObjectReference{ + APIVersion: kbappsv1.SchemeBuilder.GroupVersion.String(), + Kind: kbappsv1.ComponentKind, + Namespace: secondaries[1].Namespace, + Name: secondaries[1].Name, + UID: secondaries[1].UID, + ResourceVersion: secondaries[1].ResourceVersion, + }, + }, + }, + } + + objects, err := getObjectsFromTree(tree, store, scheme.Scheme) + Expect(err).Should(BeNil()) + Expect(objects).Should(HaveLen(3)) + expectedObjects := make(map[model.GVKNObjKey]client.Object, len(objects)) + for _, object := range []client.Object{primary, &secondaries[0], &secondaries[1]} { + objectRef, err := getObjectRef(object, scheme.Scheme) + Expect(err).Should(BeNil()) + expectedObjects[*objectRef] = object + } + for key, object := range expectedObjects { + v, ok := objects[key] + Expect(ok).Should(BeTrue()) + Expect(v).Should(Equal(object)) + } + }) + }) + + Context("getObjectTreeWithRevision", func() { + It("should work well", func() { + revision := parseRevision(resourceVersion) + tree, err := getObjectTreeWithRevision(primary, getKBOwnershipRules(), store, revision, scheme.Scheme) + Expect(err).Should(BeNil()) + Expect(tree).ShouldNot(BeNil()) + Expect(tree.Primary).Should(Equal(corev1.ObjectReference{ + APIVersion: kbappsv1.SchemeBuilder.GroupVersion.String(), + Kind: kbappsv1.ClusterKind, + Namespace: primary.Namespace, + Name: primary.Name, + UID: primary.UID, + ResourceVersion: primary.ResourceVersion, + })) + Expect(tree.Secondaries).Should(HaveLen(2)) + for i := 0; i < len(secondaries); i++ { + Expect(tree.Secondaries[i].Primary).Should(Equal(corev1.ObjectReference{ + APIVersion: kbappsv1.SchemeBuilder.GroupVersion.String(), + Kind: kbappsv1.ComponentKind, + Namespace: secondaries[i].Namespace, + Name: secondaries[i].Name, + UID: secondaries[i].UID, + ResourceVersion: secondaries[i].ResourceVersion, + })) + } + }) + }) + + Context("deleteUnusedRevisions", func() { + It("should work well", func() { + changes := []tracev1.ObjectChange{{ + ObjectReference: corev1.ObjectReference{ + APIVersion: kbappsv1.SchemeBuilder.GroupVersion.String(), + Kind: kbappsv1.ClusterKind, + Namespace: primary.GetNamespace(), + Name: primary.GetName(), + UID: primary.GetUID(), + ResourceVersion: primary.GetResourceVersion(), + }, + Revision: parseRevision(primary.GetResourceVersion()), + }} + deleteUnusedRevisions(store, changes, reference) + Expect(store.List(&schema.GroupVersionKind{ + Group: kbappsv1.GroupVersion.Group, + Version: kbappsv1.GroupVersion.Version, + Kind: kbappsv1.ClusterKind, + })).Should(HaveLen(0)) + }) + }) + }) +}) diff --git a/controllers/workloads/instanceset_controller.go b/controllers/workloads/instanceset_controller.go index c723e61245a..ea4966d3cd0 100644 --- a/controllers/workloads/instanceset_controller.go +++ b/controllers/workloads/instanceset_controller.go @@ -110,7 +110,7 @@ func (r *InstanceSetReconciler) SetupWithManager(mgr ctrl.Manager, multiClusterM } func (r *InstanceSetReconciler) setupWithManager(mgr ctrl.Manager, ctx *handler.FinderContext) error { - itsFinder := handler.NewLabelFinder(&workloads.InstanceSet{}, instanceset.WorkloadsManagedByLabelKey, workloads.Kind, instanceset.WorkloadsInstanceLabelKey) + itsFinder := handler.NewLabelFinder(&workloads.InstanceSet{}, instanceset.WorkloadsManagedByLabelKey, workloads.InstanceSetKind, instanceset.WorkloadsInstanceLabelKey) podHandler := handler.NewBuilder(ctx).AddFinder(itsFinder).Build() return intctrlutil.NewNamespacedControllerManagedBy(mgr). For(&workloads.InstanceSet{}). diff --git a/deploy/helm/config/rbac/role.yaml b/deploy/helm/config/rbac/role.yaml index 0c3fac2cabf..a6e1f3dde58 100644 --- a/deploy/helm/config/rbac/role.yaml +++ b/deploy/helm/config/rbac/role.yaml @@ -937,6 +937,32 @@ rules: - get - list - watch +- apiGroups: + - trace.kubeblocks.io + resources: + - reconciliationtraces + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - trace.kubeblocks.io + resources: + - reconciliationtraces/finalizers + verbs: + - update +- apiGroups: + - trace.kubeblocks.io + resources: + - reconciliationtraces/status + verbs: + - get + - patch + - update - apiGroups: - workloads.kubeblocks.io resources: diff --git a/deploy/helm/crds/trace.kubeblocks.io_reconciliationtraces.yaml b/deploy/helm/crds/trace.kubeblocks.io_reconciliationtraces.yaml new file mode 100644 index 00000000000..fc42f210251 --- /dev/null +++ b/deploy/helm/crds/trace.kubeblocks.io_reconciliationtraces.yaml @@ -0,0 +1,946 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + labels: + app.kubernetes.io/name: kubeblocks + name: reconciliationtraces.trace.kubeblocks.io +spec: + group: trace.kubeblocks.io + names: + categories: + - kubeblocks + - all + kind: ReconciliationTrace + listKind: ReconciliationTraceList + plural: reconciliationtraces + shortNames: + - trace + singular: reconciliationtrace + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + - description: Target Object Namespace + jsonPath: .spec.targetObject.namespace + name: TARGET_NS + type: string + - description: Target Object Name + jsonPath: .spec.targetObject.name + name: TARGET_NAME + type: string + - description: Latest Changed Object API Version + jsonPath: .status.currentState.changes[-1].objectReference.apiVersion + name: API_VERSION + type: string + - description: Latest Changed Object Kind + jsonPath: .status.currentState.changes[-1].objectReference.kind + name: KIND + type: string + - description: Latest Changed Object Namespace + jsonPath: .status.currentState.changes[-1].objectReference.namespace + name: NAMESPACE + type: string + - description: Latest Changed Object Name + jsonPath: .status.currentState.changes[-1].objectReference.name + name: NAME + type: string + - description: Latest Change Description + jsonPath: .status.currentState.changes[-1].description + name: CHANGE + type: string + name: v1 + schema: + openAPIV3Schema: + description: ReconciliationTrace is the Schema for the reconciliationtraces + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ReconciliationTraceSpec defines the desired state of ReconciliationTrace + properties: + dryRun: + description: |- + DryRun tells the Controller to simulate the reconciliation process with a new desired spec of the TargetObject. + And a reconciliation plan will be generated and described in the ReconciliationTraceStatus. + The plan generation process will not impact the state of the TargetObject. + properties: + desiredSpec: + description: |- + DesiredSpec specifies the desired spec of the TargetObject. + The desired spec will be merged into the current spec by a strategic merge patch way to build the final spec, + and the reconciliation plan will be calculated by comparing the current spec to the final spec. + DesiredSpec should be a valid YAML string. + type: string + required: + - desiredSpec + type: object + locale: + description: Locale specifies the locale to use when localizing the + reconciliation trace. + type: string + stateEvaluationExpression: + description: |- + StateEvaluationExpression specifies the state evaluation expression used during reconciliation progress observation. + The whole reconciliation process from the creation of the TargetObject to the deletion of it + is separated into several reconciliation cycles. + The StateEvaluationExpression is applied to the TargetObject, + and an evaluation result of true indicates the end of a reconciliation cycle. + StateEvaluationExpression overrides the builtin default value. + properties: + celExpression: + description: |- + CELExpression specifies to use CEL to evaluation the object state. + The root object used in the expression is the primary object. + properties: + expression: + description: Expression specifies the CEL expression. + type: string + required: + - expression + type: object + type: object + targetObject: + description: |- + TargetObject specifies the target Cluster object. + Default is the Cluster object with same namespace and name as this ReconciliationTrace object. + properties: + name: + description: |- + Name of the referent. + Default is same as the ReconciliationTrace object. + type: string + namespace: + description: |- + Namespace of the referent. + Default is same as the ReconciliationTrace object. + type: string + type: object + type: object + status: + description: ReconciliationTraceStatus defines the observed state of ReconciliationTrace + properties: + currentState: + description: |- + CurrentState is the current state of the latest reconciliation cycle, + that is the reconciliation process from the end of last reconciliation cycle until now. + properties: + changes: + description: Changes describes the detail reconciliation process. + items: + description: ObjectChange defines a detailed change of an object. + properties: + changeType: + description: |- + ChangeType specifies the change type. + Event - specifies that this is a Kubernetes Event. + Creation - specifies that this is an object creation. + Update - specifies that this is an object update. + Deletion - specifies that this is an object deletion. + enum: + - Event + - Creation + - Update + - Deletion + type: string + description: + description: Description describes the change in a user-friendly + way. + type: string + eventAttributes: + description: EventAttributes specifies the attributes of + the event when ChangeType is Event. + properties: + name: + description: Name of the Event. + type: string + reason: + description: Reason of the Event. + type: string + type: + description: Type of the Event. + type: string + required: + - name + - reason + - type + type: object + localDescription: + description: |- + LocalDescription is the localized version of Description by using the Locale specified in `spec.locale`. + Empty if the `spec.locale` is not specified. + type: string + objectReference: + description: ObjectReference specifies the Object this change + described. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + revision: + description: |- + Revision specifies the revision of the object after this change. + Revision can be compared globally between all ObjectChanges of all Objects, to build a total order object change sequence. + format: int64 + type: integer + timestamp: + description: |- + Timestamp is a timestamp representing the ReconciliationTrace Controller time when this change occurred. + It is not guaranteed to be set in happens-before order across separate changes. + It is represented in RFC3339 form and is in UTC. + format: date-time + type: string + required: + - changeType + - description + - objectReference + - revision + type: object + type: array + objectTree: + description: |- + ObjectTree specifies the current object tree of the reconciliation cycle. + Ideally, ObjectTree should be same as applying Changes to InitialObjectTree. + properties: + primary: + description: Primary specifies reference of the primary object. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + secondaries: + description: Secondaries describes all the secondary objects + of this object, if any. + x-kubernetes-preserve-unknown-fields: true + required: + - primary + type: object + summary: + description: Summary summarizes the ObjectTree and Changes. + properties: + objectSummaries: + description: ObjectSummaries summarizes each object type. + items: + description: ObjectSummary defines the total and change + of an object. + properties: + changeSummary: + description: |- + ChangeSummary summarizes the change by comparing the final state to the current state of this type. + Nil means no change. + properties: + added: + description: Added specifies the number of object + will be added. + format: int32 + type: integer + deleted: + description: Deleted specifies the number of object + will be deleted. + format: int32 + type: integer + updated: + description: Updated specifies the number of object + will be updated. + format: int32 + type: integer + type: object + objectType: + description: ObjectType of the object. + properties: + apiVersion: + description: APIVersion of the type. + type: string + kind: + description: Kind of the type. + type: string + required: + - apiVersion + - kind + type: object + total: + description: Total number of the object of type defined + by ObjectType. + format: int32 + type: integer + required: + - objectType + - total + type: object + type: array + required: + - objectSummaries + type: object + required: + - changes + - objectTree + - summary + type: object + desiredState: + description: DesiredState is the desired state of the latest reconciliation + cycle. + properties: + changes: + description: Changes describes the detail reconciliation process. + items: + description: ObjectChange defines a detailed change of an object. + properties: + changeType: + description: |- + ChangeType specifies the change type. + Event - specifies that this is a Kubernetes Event. + Creation - specifies that this is an object creation. + Update - specifies that this is an object update. + Deletion - specifies that this is an object deletion. + enum: + - Event + - Creation + - Update + - Deletion + type: string + description: + description: Description describes the change in a user-friendly + way. + type: string + eventAttributes: + description: EventAttributes specifies the attributes of + the event when ChangeType is Event. + properties: + name: + description: Name of the Event. + type: string + reason: + description: Reason of the Event. + type: string + type: + description: Type of the Event. + type: string + required: + - name + - reason + - type + type: object + localDescription: + description: |- + LocalDescription is the localized version of Description by using the Locale specified in `spec.locale`. + Empty if the `spec.locale` is not specified. + type: string + objectReference: + description: ObjectReference specifies the Object this change + described. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + revision: + description: |- + Revision specifies the revision of the object after this change. + Revision can be compared globally between all ObjectChanges of all Objects, to build a total order object change sequence. + format: int64 + type: integer + timestamp: + description: |- + Timestamp is a timestamp representing the ReconciliationTrace Controller time when this change occurred. + It is not guaranteed to be set in happens-before order across separate changes. + It is represented in RFC3339 form and is in UTC. + format: date-time + type: string + required: + - changeType + - description + - objectReference + - revision + type: object + type: array + objectTree: + description: |- + ObjectTree specifies the current object tree of the reconciliation cycle. + Ideally, ObjectTree should be same as applying Changes to InitialObjectTree. + properties: + primary: + description: Primary specifies reference of the primary object. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + secondaries: + description: Secondaries describes all the secondary objects + of this object, if any. + x-kubernetes-preserve-unknown-fields: true + required: + - primary + type: object + summary: + description: Summary summarizes the ObjectTree and Changes. + properties: + objectSummaries: + description: ObjectSummaries summarizes each object type. + items: + description: ObjectSummary defines the total and change + of an object. + properties: + changeSummary: + description: |- + ChangeSummary summarizes the change by comparing the final state to the current state of this type. + Nil means no change. + properties: + added: + description: Added specifies the number of object + will be added. + format: int32 + type: integer + deleted: + description: Deleted specifies the number of object + will be deleted. + format: int32 + type: integer + updated: + description: Updated specifies the number of object + will be updated. + format: int32 + type: integer + type: object + objectType: + description: ObjectType of the object. + properties: + apiVersion: + description: APIVersion of the type. + type: string + kind: + description: Kind of the type. + type: string + required: + - apiVersion + - kind + type: object + total: + description: Total number of the object of type defined + by ObjectType. + format: int32 + type: integer + required: + - objectType + - total + type: object + type: array + required: + - objectSummaries + type: object + required: + - changes + - objectTree + - summary + type: object + dryRunResult: + description: DryRunResult specifies the dry-run result. + properties: + desiredSpecRevision: + description: DesiredSpecRevision specifies the revision of the + DesiredSpec. + type: string + message: + description: Message specifies a description of the failure reason. + type: string + observedTargetGeneration: + description: ObservedTargetGeneration specifies the observed generation + of the TargetObject. + format: int64 + type: integer + phase: + description: |- + Phase specifies the current phase of the plan generation process. + Succeed - the plan is calculated successfully. + Failed - the plan can't be generated for some reason described in Reason. + enum: + - Succeed + - Failed + type: string + plan: + description: Plan describes the detail reconciliation process + if the DesiredSpec is applied. + properties: + changes: + description: Changes describes the detail reconciliation process. + items: + description: ObjectChange defines a detailed change of an + object. + properties: + changeType: + description: |- + ChangeType specifies the change type. + Event - specifies that this is a Kubernetes Event. + Creation - specifies that this is an object creation. + Update - specifies that this is an object update. + Deletion - specifies that this is an object deletion. + enum: + - Event + - Creation + - Update + - Deletion + type: string + description: + description: Description describes the change in a user-friendly + way. + type: string + eventAttributes: + description: EventAttributes specifies the attributes + of the event when ChangeType is Event. + properties: + name: + description: Name of the Event. + type: string + reason: + description: Reason of the Event. + type: string + type: + description: Type of the Event. + type: string + required: + - name + - reason + - type + type: object + localDescription: + description: |- + LocalDescription is the localized version of Description by using the Locale specified in `spec.locale`. + Empty if the `spec.locale` is not specified. + type: string + objectReference: + description: ObjectReference specifies the Object this + change described. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + revision: + description: |- + Revision specifies the revision of the object after this change. + Revision can be compared globally between all ObjectChanges of all Objects, to build a total order object change sequence. + format: int64 + type: integer + timestamp: + description: |- + Timestamp is a timestamp representing the ReconciliationTrace Controller time when this change occurred. + It is not guaranteed to be set in happens-before order across separate changes. + It is represented in RFC3339 form and is in UTC. + format: date-time + type: string + required: + - changeType + - description + - objectReference + - revision + type: object + type: array + objectTree: + description: |- + ObjectTree specifies the current object tree of the reconciliation cycle. + Ideally, ObjectTree should be same as applying Changes to InitialObjectTree. + properties: + primary: + description: Primary specifies reference of the primary + object. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + secondaries: + description: Secondaries describes all the secondary objects + of this object, if any. + x-kubernetes-preserve-unknown-fields: true + required: + - primary + type: object + summary: + description: Summary summarizes the ObjectTree and Changes. + properties: + objectSummaries: + description: ObjectSummaries summarizes each object type. + items: + description: ObjectSummary defines the total and change + of an object. + properties: + changeSummary: + description: |- + ChangeSummary summarizes the change by comparing the final state to the current state of this type. + Nil means no change. + properties: + added: + description: Added specifies the number of object + will be added. + format: int32 + type: integer + deleted: + description: Deleted specifies the number of + object will be deleted. + format: int32 + type: integer + updated: + description: Updated specifies the number of + object will be updated. + format: int32 + type: integer + type: object + objectType: + description: ObjectType of the object. + properties: + apiVersion: + description: APIVersion of the type. + type: string + kind: + description: Kind of the type. + type: string + required: + - apiVersion + - kind + type: object + total: + description: Total number of the object of type + defined by ObjectType. + format: int32 + type: integer + required: + - objectType + - total + type: object + type: array + required: + - objectSummaries + type: object + required: + - changes + - objectTree + - summary + type: object + reason: + description: Reason specifies the reason when the Phase is Failed. + type: string + specDiff: + description: "SpecDiff describes the diff between the current + spec and the final spec.\nThe whole spec struct will be compared + and an example SpecDiff looks like:\n{\n \tAffinity: {\n \t\tPodAntiAffinity: + \"Preferred\",\n \t\tTenancy: \"SharedNode\",\n \t},\n \tComponentSpecs: + {\n \t\t{\n \t\t\tComponentDef: \"postgresql\",\n \t\t\tName: + \"postgresql\",\n-\t\t\tReplicas: 2,\n \t\t\tResources:\n \t\t\t{\n + \t\t\t\tLimits:\n \t\t\t\t{\n-\t\t\t\t\tCPU: 500m,\n-\t\t\t\t\tMemory: + 512Mi,\n \t\t\t\t},\n \t\t\t\tRequests:\n \t\t\t\t{\n-\t\t\t\t\tCPU: + 500m,\n-\t\t\t\t\tMemory: 512Mi,\n \t\t\t\t},\n \t\t\t},\n \t\t},\n + \t},\n}" + type: string + required: + - desiredSpecRevision + - observedTargetGeneration + - plan + - specDiff + type: object + initialObjectTree: + description: InitialObjectTree specifies the initial object tree when + the latest reconciliation cycle started. + properties: + primary: + description: Primary specifies reference of the primary object. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + TODO: this design is not final and this field is subject to change in the future. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + secondaries: + description: Secondaries describes all the secondary objects of + this object, if any. + x-kubernetes-preserve-unknown-fields: true + required: + - primary + type: object + required: + - currentState + - initialObjectTree + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/helm/templates/_helpers.tpl b/deploy/helm/templates/_helpers.tpl index dceb599f0a3..928983a485c 100644 --- a/deploy/helm/templates/_helpers.tpl +++ b/deploy/helm/templates/_helpers.tpl @@ -336,4 +336,8 @@ Define the replica count for kubeblocks. {{- else }} {{- .Values.replicaCount }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} + +{{- define "kubeblocks.i18nResourcesName" -}} +{{ include "kubeblocks.fullname" . }}-i18n-resources +{{- end }} diff --git a/deploy/helm/templates/deployment.yaml b/deploy/helm/templates/deployment.yaml index f80deaf40b5..e67d5821ea8 100644 --- a/deploy/helm/templates/deployment.yaml +++ b/deploy/helm/templates/deployment.yaml @@ -72,6 +72,7 @@ spec: - "--operations={{ ne .Values.controllers.operations.enabled false }}" - "--extensions={{- default "true" ( include "kubeblocks.addonControllerEnabled" . ) }}" - "--experimental={{- default "false" .Values.controllers.experimental.enabled }}" + - "--trace={{- default "true" .Values.controllers.trace.enabled }}" {{- with .Values.managedNamespaces }} - "--managed-namespaces={{ . }}" {{- end }} @@ -183,6 +184,10 @@ spec: value: {{ .Values.featureGates.componentReplicasAnnotation.enabled | quote }} - name: IN_PLACE_POD_VERTICAL_SCALING value: {{ .Values.featureGates.inPlacePodVerticalScaling.enabled | quote }} + {{- if .Values.controllers.trace.enabled }} + - name: I18N_RESOURCES_NAME + value: {{ include "kubeblocks.i18nResourcesName" . }} + {{- end }} {{- if .Values.extraEnvs }} {{- toYaml .Values.extraEnvs | nindent 12 }} {{- end }} diff --git a/deploy/helm/templates/i18n-configmap.yaml b/deploy/helm/templates/i18n-configmap.yaml new file mode 100644 index 00000000000..423abe1c955 --- /dev/null +++ b/deploy/helm/templates/i18n-configmap.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "kubeblocks.i18nResourcesName" . }} + labels: + {{- include "kubeblocks.labels" . | nindent 4 }} +data: + en: | + v1/ConfigMap/Creation=configuration file %s/%s is created. + v1/Pod/Creation=Pod %s/%s is created. + v1/Pod/Update=Pod %s/%s is updated. + zh_CN: | + v1/ConfigMap/Creation=配置文件 %s/%s 创建成功。 + v1/Pod/Creation=Pod %s/%s 创建成功。 + v1/Pod/Update=Pod %s/%s 更新成功。 diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index a40bcbfb848..32b4398b5d5 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -1865,6 +1865,8 @@ controllers: enabled: true experimental: enabled: false + trace: + enabled: false featureGates: ignoreConfigTemplateDefaultMode: diff --git a/go.mod b/go.mod index a677c149e00..83ed2628c2c 100644 --- a/go.mod +++ b/go.mod @@ -15,12 +15,14 @@ require ( github.com/clbanning/mxj/v2 v2.5.7 github.com/docker/docker v25.0.6+incompatible github.com/evanphx/json-patch v5.6.0+incompatible + github.com/evanphx/json-patch/v5 v5.8.0 github.com/fasthttp/router v1.4.20 github.com/fsnotify/fsnotify v1.7.0 github.com/go-logr/logr v1.4.1 github.com/go-logr/zapr v1.3.0 github.com/go-sql-driver/mysql v1.7.1 github.com/golang/mock v1.6.0 + github.com/google/cel-go v0.17.8 github.com/google/go-cmp v0.6.0 github.com/imdario/mergo v0.3.14 github.com/jinzhu/copier v0.4.0 @@ -114,7 +116,6 @@ require ( github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/emicklei/proto v1.10.0 // indirect - github.com/evanphx/json-patch/v5 v5.8.0 // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect github.com/fatih/camelcase v1.0.0 // indirect github.com/fatih/color v1.16.0 // indirect @@ -133,7 +134,6 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.2 // indirect - github.com/google/cel-go v0.17.8 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 // indirect diff --git a/pkg/constant/const.go b/pkg/constant/const.go index ec8c3d915c2..fa807c7d55e 100644 --- a/pkg/constant/const.go +++ b/pkg/constant/const.go @@ -34,10 +34,20 @@ const ( ) const ( - StatefulSetKind = "StatefulSet" - PodKind = "Pod" - JobKind = "Job" - VolumeSnapshotKind = "VolumeSnapshot" + StatefulSetKind = "StatefulSet" + PodKind = "Pod" + JobKind = "Job" + VolumeSnapshotKind = "VolumeSnapshot" + ServiceKind = "Service" + SecretKind = "Secret" + ConfigMapKind = "ConfigMap" + PersistentVolumeClaimKind = "PersistentVolumeClaim" + PersistentVolumeKind = "PersistentVolume" + ConfigurationKind = "Configuration" + ClusterRoleBindingKind = "ClusterRoleBinding" + RoleBindingKind = "RoleBinding" + ServiceAccountKind = "ServiceAccount" + EventKind = "Event" ) // username and password are keys in created secrets for others to refer to. @@ -67,3 +77,8 @@ const ( const InvalidContainerPort int32 = 0 const EmptyInsTemplateName = "" + +type Key string + +// DryRunContextKey tells the KB Controllers to do dry-run reconciliations +const DryRunContextKey Key = "dry-run" diff --git a/pkg/constant/viper_config.go b/pkg/constant/viper_config.go index d3b7fe6e1ab..2dc51c11e20 100644 --- a/pkg/constant/viper_config.go +++ b/pkg/constant/viper_config.go @@ -52,4 +52,6 @@ const ( CfgKBReconcileWorkers = "KUBEBLOCKS_RECONCILE_WORKERS" CfgClientQPS = "CLIENT_QPS" CfgClientBurst = "CLIENT_BURST" + + I18nResourcesName = "I18N_RESOURCES_NAME" ) diff --git a/pkg/controller/builder/builder_cluster.go b/pkg/controller/builder/builder_cluster.go index e820a1c7c01..e0e061718b3 100644 --- a/pkg/controller/builder/builder_cluster.go +++ b/pkg/controller/builder/builder_cluster.go @@ -37,3 +37,8 @@ func (builder *ClusterBuilder) SetComponentSpecs(specs []appsv1.ClusterComponent builder.get().Spec.ComponentSpecs = specs return builder } + +func (builder *ClusterBuilder) SetResourceVersion(resourceVersion string) *ClusterBuilder { + builder.get().ResourceVersion = resourceVersion + return builder +} diff --git a/pkg/controller/graph/dag.go b/pkg/controller/graph/dag.go index 5d9d6d4b451..736c86944f3 100644 --- a/pkg/controller/graph/dag.go +++ b/pkg/controller/graph/dag.go @@ -146,7 +146,7 @@ func (d *DAG) AddConnectRoot(v Vertex) bool { // WalkTopoOrder walks the DAG 'd' in topology order func (d *DAG) WalkTopoOrder(walkFunc WalkFunc, less func(v1, v2 Vertex) bool) error { - if err := d.validate(); err != nil { + if err := d.Validate(); err != nil { return err } orders := d.topologicalOrder(false, less) @@ -160,7 +160,7 @@ func (d *DAG) WalkTopoOrder(walkFunc WalkFunc, less func(v1, v2 Vertex) bool) er // WalkReverseTopoOrder walks the DAG 'd' in reverse topology order func (d *DAG) WalkReverseTopoOrder(walkFunc WalkFunc, less func(v1, v2 Vertex) bool) error { - if err := d.validate(); err != nil { + if err := d.Validate(); err != nil { return err } orders := d.topologicalOrder(true, less) @@ -178,7 +178,7 @@ func (d *DAG) WalkBFS(walkFunc WalkFunc) error { } func (d *DAG) bfs(walkFunc WalkFunc, less func(v1, v2 Vertex) bool) error { - if err := d.validate(); err != nil { + if err := d.Validate(); err != nil { return err } queue := make([]Vertex, 0) @@ -280,8 +280,8 @@ func (d *DAG) String() string { return str } -// validate 'd' has single Root and has no cycles -func (d *DAG) validate() error { +// Validate 'd' has single Root and has no cycles +func (d *DAG) Validate() error { // single Root validation root := d.Root() if root == nil { diff --git a/pkg/controller/graph/dag_test.go b/pkg/controller/graph/dag_test.go index 87f3cb7e078..4003372d7a8 100644 --- a/pkg/controller/graph/dag_test.go +++ b/pkg/controller/graph/dag_test.go @@ -203,7 +203,7 @@ func TestWalkBFS(t *testing.T) { func TestValidate(t *testing.T) { dag := NewDAG() - err := dag.validate() + err := dag.Validate() if err == nil { t.Error("nil root not found") } @@ -217,7 +217,7 @@ func TestValidate(t *testing.T) { dag.Connect(1, 2) dag.Connect(2, 3) dag.Connect(3, 1) - err = dag.validate() + err = dag.Validate() if err == nil { t.Error("cycle not found") } @@ -225,7 +225,7 @@ func TestValidate(t *testing.T) { t.Error("error not as expected") } dag.Connect(1, 1) - err = dag.validate() + err = dag.Validate() if err == nil { t.Error("self-cycle not found") } diff --git a/pkg/controller/instanceset/suite_test.go b/pkg/controller/instanceset/suite_test.go index 60cbfabf695..54d275ae337 100644 --- a/pkg/controller/instanceset/suite_test.go +++ b/pkg/controller/instanceset/suite_test.go @@ -71,7 +71,7 @@ var ( selectors = map[string]string{ constant.AppInstanceLabelKey: name, - WorkloadsManagedByLabelKey: workloads.Kind, + WorkloadsManagedByLabelKey: workloads.InstanceSetKind, } roles = []workloads.ReplicaRole{ { diff --git a/pkg/controller/instanceset/utils.go b/pkg/controller/instanceset/utils.go index f60dfd80559..d3cd1817586 100644 --- a/pkg/controller/instanceset/utils.go +++ b/pkg/controller/instanceset/utils.go @@ -212,7 +212,7 @@ func mergeMap[K comparable, V any](src, dst *map[K]V) { func getMatchLabels(name string) map[string]string { return map[string]string{ constant.AppManagedByLabelKey: constant.AppName, - WorkloadsManagedByLabelKey: workloads.Kind, + WorkloadsManagedByLabelKey: workloads.InstanceSetKind, WorkloadsInstanceLabelKey: name, } } diff --git a/pkg/controllerutil/util.go b/pkg/controllerutil/util.go index dd01b9ba6b5..6d1f0cd361d 100644 --- a/pkg/controllerutil/util.go +++ b/pkg/controllerutil/util.go @@ -163,7 +163,7 @@ func SetControllerReference(owner, object metav1.Object) error { return controllerutil.SetControllerReference(owner, object, innerScheme) } -func GeKubeRestConfig(userAgent string) *rest.Config { +func GetKubeRestConfig(userAgent string) *rest.Config { cfg := ctrl.GetConfigOrDie() clientQPS := viper.GetInt(constant.CfgClientQPS) if clientQPS != 0 { diff --git a/pkg/kbagent/client/http_client.go b/pkg/kbagent/client/http_client.go index 413b11217f4..c7d135a771e 100644 --- a/pkg/kbagent/client/http_client.go +++ b/pkg/kbagent/client/http_client.go @@ -27,6 +27,7 @@ import ( "io" "net/http" + "github.com/apecloud/kubeblocks/pkg/constant" "github.com/apecloud/kubeblocks/pkg/kbagent/proto" ) @@ -45,6 +46,11 @@ var _ Client = &httpClient{} func (c *httpClient) Action(ctx context.Context, req proto.ActionRequest) (proto.ActionResponse, error) { rsp := proto.ActionResponse{} + dryRun, ok := ctx.Value(constant.DryRunContextKey).(bool) + if ok && dryRun { + return rsp, nil + } + data, err := json.Marshal(req) if err != nil { return rsp, err diff --git a/pkg/testutil/apps/cluster_instance_set_test_util.go b/pkg/testutil/apps/cluster_instance_set_test_util.go index 45492ba91db..f15f5952be2 100644 --- a/pkg/testutil/apps/cluster_instance_set_test_util.go +++ b/pkg/testutil/apps/cluster_instance_set_test_util.go @@ -156,11 +156,11 @@ func MockInstanceSetPod( name = its.Name } ml := map[string]string{ - "workloads.kubeblocks.io/managed-by": workloads.Kind, + "workloads.kubeblocks.io/managed-by": workloads.InstanceSetKind, "workloads.kubeblocks.io/instance": name, } podFactory := NewPodFactory(testCtx.DefaultNamespace, podName). - SetOwnerReferences(workloads.GroupVersion.String(), workloads.Kind, its). + SetOwnerReferences(workloads.GroupVersion.String(), workloads.InstanceSetKind, its). AddAppInstanceLabel(clusterName). AddAppComponentLabel(consensusCompName). AddAppManagedByLabel().