diff --git a/Makefile b/Makefile index 0cd9df6a..3bfbddfd 100644 --- a/Makefile +++ b/Makefile @@ -164,7 +164,7 @@ GOVULNCHECK ?= $(LOCALBIN)/govulncheck ## Tool Versions KUSTOMIZE_VERSION ?= v5.0.1 CONTROLLER_TOOLS_VERSION ?= v0.12.0 -GOLANGCI_LINT_VERSION ?= v1.53.3 +GOLANGCI_LINT_VERSION ?= v1.55.2 MOCKGEN_VERSION ?= v0.2.0 GORELEASER_VERSION ?= v1.21.0 MDTOC_VERSION ?= v1.1.0 diff --git a/PROJECT b/PROJECT index 14e04202..3608e28c 100644 --- a/PROJECT +++ b/PROJECT @@ -62,4 +62,13 @@ resources: defaulting: true validation: true webhookVersion: v1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: mercedes-benz.com + group: garm-operator + kind: Runner + path: github.com/mercedes-benz/garm-operator/api/v1alpha1 + version: v1alpha1 version: "3" diff --git a/Tiltfile b/Tiltfile index 489c32a0..fc72dbd8 100644 --- a/Tiltfile +++ b/Tiltfile @@ -26,6 +26,7 @@ k8s_resource( 'images.garm-operator.mercedes-benz.com:customresourcedefinition', 'organizations.garm-operator.mercedes-benz.com:customresourcedefinition', 'pools.garm-operator.mercedes-benz.com:customresourcedefinition', + 'runners.garm-operator.mercedes-benz.com:customresourcedefinition', 'repositories.garm-operator.mercedes-benz.com:customresourcedefinition', 'garm-operator-controller-manager:serviceaccount', 'garm-operator-leader-election-role:role', diff --git a/api/v1alpha1/pool_types.go b/api/v1alpha1/pool_types.go index 0c871909..aa9f9c3f 100644 --- a/api/v1alpha1/pool_types.go +++ b/api/v1alpha1/pool_types.go @@ -109,6 +109,12 @@ func MatchesGitHubScope(name, kind string) Predicate { } } +func MatchesID(id string) Predicate { + return func(p Pool) bool { + return p.Status.ID == id + } +} + func (p *PoolList) FilterByFields(predicates ...Predicate) { var filteredItems []Pool diff --git a/api/v1alpha1/runner_types.go b/api/v1alpha1/runner_types.go new file mode 100644 index 00000000..6212a737 --- /dev/null +++ b/api/v1alpha1/runner_types.go @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: MIT + +package v1alpha1 + +import ( + commonParams "github.com/cloudbase/garm-provider-common/params" + "github.com/cloudbase/garm/params" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// RunnerSpec defines the desired state of Runner +type RunnerSpec struct{} + +// RunnerStatus defines the observed state of Runner +type RunnerStatus struct { + // ID is the database ID of this instance. + ID string `json:"id,omitempty"` + + // PeoviderID is the unique ID the provider associated + // with the compute instance. We use this to identify the + // instance in the provider. + ProviderID string `json:"providerId,omitempty"` + + // AgentID is the github runner agent ID. + AgentID int64 `json:"agentId"` + + // Name is the name associated with an instance. Depending on + // the provider, this may or may not be useful in the context of + // the provider, but we can use it internally to identify the + // instance. + Name string `json:"name,omitempty"` + + // OSType is the operating system type. For now, only Linux and + // Windows are supported. + OSType commonParams.OSType `json:"osType,omitempty"` + + // OSName is the name of the OS. Eg: ubuntu, centos, etc. + OSName string `json:"osName,omitempty"` + + // OSVersion is the version of the operating system. + OSVersion string `json:"osVersion,omitempty"` + + // OSArch is the operating system architecture. + OSArch commonParams.OSArch `json:"osArch,omitempty"` + + // Addresses is a list of IP addresses the provider reports + // for this instance. + Addresses []commonParams.Address `json:"addresses,omitempty"` + + // Status is the status of the instance inside the provider (eg: running, stopped, etc) + Status commonParams.InstanceStatus `json:"status,omitempty"` + + // RunnerStatus is the github runner status as it appears on GitHub. + InstanceStatus params.RunnerStatus `json:"instanceStatus,omitempty"` + + // PoolID is the ID of the garm pool to which a runner belongs. + PoolID string `json:"poolId,omitempty"` + + // ProviderFault holds any error messages captured from the IaaS provider that is + // responsible for managing the lifecycle of the runner. + ProviderFault string `json:"providerFault,omitempty"` + + // StatusMessages is a list of status messages sent back by the runner as it sets itself + // up. + + //// UpdatedAt is the timestamp of the last update to this runner. + // UpdatedAt time.Time `json:"updated_at"` + + // GithubRunnerGroup is the github runner group to which the runner belongs. + // The runner group must be created by someone with access to the enterprise. + GitHubRunnerGroup string `json:"githubRunnerGroup"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:resource:path=runners,scope=Namespaced,categories=garm,shortName=run +//+kubebuilder:subresource:status +//+kubebuilder:printcolumn:name="ID",type="string",JSONPath=".status.id",description="Runner ID" +//+kubebuilder:printcolumn:name="Pool",type="string",JSONPath=".status.poolId",description="Pool CR Name" +//+kubebuilder:printcolumn:name="Garm Runner Status",type="string",JSONPath=".status.status",description="Garm Runner Status" +//+kubebuilder:printcolumn:name="Provider Runner Status",type="string",JSONPath=".status.instanceStatus",description="Provider Runner Status" +//+kubebuilder:printcolumn:name="Provider ID",type="string",JSONPath=".status.providerId",description="Provider ID",priority=1 +//+kubebuilder:printcolumn:name="Agent ID",type="string",JSONPath=".status.agentId",description="Agent ID",priority=1 +//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// Runner is the Schema for the runners API +type Runner struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec RunnerSpec `json:"spec,omitempty"` + Status RunnerStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// RunnerList contains a list of Runner +type RunnerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Runner `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Runner{}, &RunnerList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6ed52ae3..a07e1bc1 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -8,6 +8,7 @@ package v1alpha1 import ( + "github.com/cloudbase/garm-provider-common/params" "k8s.io/apimachinery/pkg/runtime" ) @@ -465,6 +466,100 @@ func (in *RepositoryStatus) DeepCopy() *RepositoryStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Runner) DeepCopyInto(out *Runner) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Runner. +func (in *Runner) DeepCopy() *Runner { + if in == nil { + return nil + } + out := new(Runner) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Runner) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RunnerList) DeepCopyInto(out *RunnerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Runner, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerList. +func (in *RunnerList) DeepCopy() *RunnerList { + if in == nil { + return nil + } + out := new(RunnerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RunnerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RunnerSpec) DeepCopyInto(out *RunnerSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerSpec. +func (in *RunnerSpec) DeepCopy() *RunnerSpec { + if in == nil { + return nil + } + out := new(RunnerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RunnerStatus) DeepCopyInto(out *RunnerStatus) { + *out = *in + if in.Addresses != nil { + in, out := &in.Addresses, &out.Addresses + *out = make([]params.Address, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RunnerStatus. +func (in *RunnerStatus) DeepCopy() *RunnerStatus { + if in == nil { + return nil + } + out := new(RunnerStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecretRef) DeepCopyInto(out *SecretRef) { *out = *in diff --git a/cmd/main.go b/cmd/main.go index 465f742f..18d8cac3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -3,8 +3,9 @@ package main import ( + "context" "fmt" - "os" + "log" "gopkg.in/yaml.v2" "k8s.io/apimachinery/pkg/runtime" @@ -14,6 +15,7 @@ import ( "k8s.io/klog/v2/klogr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/healthz" garmoperatorv1alpha1 "github.com/mercedes-benz/garm-operator/api/v1alpha1" @@ -36,6 +38,12 @@ func init() { } func main() { + if err := run(); err != nil { + log.Fatal(err) + } +} + +func run() error { ctrl.SetLogger(klogr.New()) // initiate flags @@ -46,8 +54,7 @@ func main() { // call GenerateConfig() function from config package if err := config.GenerateConfig(f, configFile); err != nil { - setupLog.Error(err, "failed to read config") - os.Exit(1) + return fmt.Errorf("failed to read config: %w", err) } // check if dry-run flag is set to true @@ -57,11 +64,10 @@ func main() { if dryRun { yamlConfig, err := yaml.Marshal(config.Config) if err != nil { - setupLog.Error(err, "failed to marshal config as yaml") - os.Exit(1) + return fmt.Errorf("failed to marshal config as yaml: %w", err) } fmt.Printf("generated Config as yaml:\n%s\n", yamlConfig) - os.Exit(0) + return nil } var watchNamespaces []string @@ -97,8 +103,7 @@ func main() { }, }) if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) + return fmt.Errorf("unable to start manager: %w", err) } ctx := ctrl.SetupSignalHandler() @@ -111,8 +116,7 @@ func main() { Password: config.Config.Garm.Password, Email: config.Config.Garm.Email, }); err != nil { - setupLog.Error(err, "failed to initialize GARM") - os.Exit(1) + return fmt.Errorf("failed to initialize GARM: %w", err) } } @@ -125,8 +129,7 @@ func main() { Username: config.Config.Garm.Username, Password: config.Config.Garm.Password, }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Enterprise") - os.Exit(1) + return fmt.Errorf("unable to create controller Enterprise: %w", err) } if err = (&controller.PoolReconciler{ Client: mgr.GetClient(), @@ -137,21 +140,17 @@ func main() { Username: config.Config.Garm.Username, Password: config.Config.Garm.Password, }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Pool") - os.Exit(1) + return fmt.Errorf("unable to create controller Pool: %w", err) } if err = (&garmoperatorv1alpha1.Pool{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "Pool") - os.Exit(1) + return fmt.Errorf("unable to create webhook Pool: %w", err) } if err = (&garmoperatorv1alpha1.Image{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "Image") - os.Exit(1) + return fmt.Errorf("unable to create webhook Image: %w", err) } if err = (&garmoperatorv1alpha1.Repository{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "Repository") - os.Exit(1) + return fmt.Errorf("unable to create webhook Repository: %w", err) } if err = (&controller.OrganizationReconciler{ @@ -163,8 +162,7 @@ func main() { Username: config.Config.Garm.Username, Password: config.Config.Garm.Password, }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Organization") - os.Exit(1) + return fmt.Errorf("unable to create controller Organization: %w", err) } if err = (&controller.RepositoryReconciler{ @@ -176,23 +174,41 @@ func main() { Username: config.Config.Garm.Username, Password: config.Config.Garm.Password, }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Repository") - os.Exit(1) + return fmt.Errorf("unable to create controller Repository: %w", err) } + + eventChan := make(chan event.GenericEvent) + runnerReconciler := &controller.RunnerReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + + BaseURL: config.Config.Garm.Server, + Username: config.Config.Garm.Username, + Password: config.Config.Garm.Password, + } + + // setup controller so it can reconcile if events from eventChan are queued + if err = runnerReconciler.SetupWithManager(mgr, eventChan); err != nil { + return fmt.Errorf("unable to create controller Runner: %w", err) + } + + // fetch runner instances periodically and enqueue reconcile events for runner ctrl if external system has changed + ctx, cancel := context.WithCancel(context.Background()) + go runnerReconciler.PollRunnerInstances(ctx, eventChan) + defer cancel() + //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") - os.Exit(1) + return fmt.Errorf("unable to set up health check: %w", err) } if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") - os.Exit(1) + return fmt.Errorf("unable to set up ready check: %w", err) } setupLog.Info("starting manager") if err := mgr.Start(ctx); err != nil { - setupLog.Error(err, "problem running manager") - os.Exit(1) + return fmt.Errorf("unable to start manager: %w", err) } + return nil } diff --git a/config/crd/bases/garm-operator.mercedes-benz.com_runners.yaml b/config/crd/bases/garm-operator.mercedes-benz.com_runners.yaml new file mode 100644 index 00000000..244eebc6 --- /dev/null +++ b/config/crd/bases/garm-operator.mercedes-benz.com_runners.yaml @@ -0,0 +1,147 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.12.0 + name: runners.garm-operator.mercedes-benz.com +spec: + group: garm-operator.mercedes-benz.com + names: + categories: + - garm + kind: Runner + listKind: RunnerList + plural: runners + shortNames: + - run + singular: runner + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Runner ID + jsonPath: .status.id + name: ID + type: string + - description: Pool CR Name + jsonPath: .status.poolId + name: Pool + type: string + - description: Garm Runner Status + jsonPath: .status.status + name: Garm Runner Status + type: string + - description: Provider Runner Status + jsonPath: .status.instanceStatus + name: Provider Runner Status + type: string + - description: Provider ID + jsonPath: .status.providerId + name: Provider ID + priority: 1 + type: string + - description: Agent ID + jsonPath: .status.agentId + name: Agent ID + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: Runner is the Schema for the runners API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RunnerSpec defines the desired state of Runner + type: object + status: + description: RunnerStatus defines the observed state of Runner + properties: + addresses: + description: Addresses is a list of IP addresses the provider reports + for this instance. + items: + properties: + address: + type: string + type: + type: string + required: + - address + - type + type: object + type: array + agentId: + description: AgentID is the github runner agent ID. + format: int64 + type: integer + githubRunnerGroup: + description: GithubRunnerGroup is the github runner group to which + the runner belongs. The runner group must be created by someone + with access to the enterprise. + type: string + id: + description: ID is the database ID of this instance. + type: string + instanceStatus: + description: RunnerStatus is the github runner status as it appears + on GitHub. + type: string + name: + description: Name is the name associated with an instance. Depending + on the provider, this may or may not be useful in the context of + the provider, but we can use it internally to identify the instance. + type: string + osArch: + description: OSArch is the operating system architecture. + type: string + osName: + description: 'OSName is the name of the OS. Eg: ubuntu, centos, etc.' + type: string + osType: + description: OSType is the operating system type. For now, only Linux + and Windows are supported. + type: string + osVersion: + description: OSVersion is the version of the operating system. + type: string + poolId: + description: PoolID is the ID of the garm pool to which a runner belongs. + type: string + providerFault: + description: ProviderFault holds any error messages captured from + the IaaS provider that is responsible for managing the lifecycle + of the runner. + type: string + providerId: + description: PeoviderID is the unique ID the provider associated with + the compute instance. We use this to identify the instance in the + provider. + type: string + status: + description: 'Status is the status of the instance inside the provider + (eg: running, stopped, etc)' + type: string + required: + - agentId + - githubRunnerGroup + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index c712d578..13048f1a 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -7,6 +7,7 @@ resources: - bases/garm-operator.mercedes-benz.com_organizations.yaml - bases/garm-operator.mercedes-benz.com_images.yaml - bases/garm-operator.mercedes-benz.com_repositories.yaml +- bases/garm-operator.mercedes-benz.com_runners.yaml #+kubebuilder:scaffold:crdkustomizeresource patches: @@ -16,6 +17,7 @@ patches: - path: patches/webhook_in_pools.yaml - path: patches/webhook_in_organizations.yaml - path: patches/webhook_in_repositories.yaml +#- path: patches/webhook_in_runners.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. @@ -25,6 +27,7 @@ patches: - path: patches/cainjection_in_organizations.yaml #- path: patches/cainjection_in_images.yaml - path: patches/cainjection_in_repositories.yaml +#- path: patches/cainjection_in_runners.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_runners.yaml b/config/crd/patches/cainjection_in_runners.yaml new file mode 100644 index 00000000..3b4c8197 --- /dev/null +++ b/config/crd/patches/cainjection_in_runners.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: CERTIFICATE_NAMESPACE/CERTIFICATE_NAME + name: runners.garm-operator.mercedes-benz.com diff --git a/config/crd/patches/webhook_in_runners.yaml b/config/crd/patches/webhook_in_runners.yaml new file mode 100644 index 00000000..45b3e44c --- /dev/null +++ b/config/crd/patches/webhook_in_runners.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: runners.garm-operator.mercedes-benz.com +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/overlays/local/manager_patch.yaml b/config/overlays/local/manager_patch.yaml index 7ea8bb76..3a9d1f0a 100644 --- a/config/overlays/local/manager_patch.yaml +++ b/config/overlays/local/manager_patch.yaml @@ -9,6 +9,8 @@ spec: containers: - name: manager args: - - --garm-server= - - --garm-username= - - --garm-password= + - --garm-server=http://garm-server.garm-server.svc.cluster.local:9997 + - --garm-username=admin + - --garm-password=LmrBG1KcBOsDfNKq4cQTGpc0hJ0kejkk + - --operator-watch-namespace=garm-operator-system + diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 1ef1cfd1..c787d4af 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -136,3 +136,29 @@ rules: - get - patch - update +- apiGroups: + - garm-operator.mercedes-benz.com + resources: + - runners + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - garm-operator.mercedes-benz.com + resources: + - runners/finalizers + verbs: + - update +- apiGroups: + - garm-operator.mercedes-benz.com + resources: + - runners/status + verbs: + - get + - patch + - update diff --git a/config/rbac/runner_editor_role.yaml b/config/rbac/runner_editor_role.yaml new file mode 100644 index 00000000..4385d588 --- /dev/null +++ b/config/rbac/runner_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit runners. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: runner-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: garm-operator + app.kubernetes.io/part-of: garm-operator + app.kubernetes.io/managed-by: kustomize + name: runner-editor-role +rules: +- apiGroups: + - garm-operator.mercedes-benz.com + resources: + - runners + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - garm-operator.mercedes-benz.com + resources: + - runners/status + verbs: + - get diff --git a/config/rbac/runner_viewer_role.yaml b/config/rbac/runner_viewer_role.yaml new file mode 100644 index 00000000..ef1ba8a8 --- /dev/null +++ b/config/rbac/runner_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view runners. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: runner-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: garm-operator + app.kubernetes.io/part-of: garm-operator + app.kubernetes.io/managed-by: kustomize + name: runner-viewer-role +rules: +- apiGroups: + - garm-operator.mercedes-benz.com + resources: + - runners + verbs: + - get + - list + - watch +- apiGroups: + - garm-operator.mercedes-benz.com + resources: + - runners/status + verbs: + - get diff --git a/config/samples/garm-operator_v1alpha1_image.yaml b/config/samples/garm-operator_v1alpha1_image.yaml index 257c208d..920c22e7 100644 --- a/config/samples/garm-operator_v1alpha1_image.yaml +++ b/config/samples/garm-operator_v1alpha1_image.yaml @@ -10,4 +10,4 @@ metadata: name: runner-default namespace: garm-operator-system spec: - tag: linux-ubuntu-22.04-arm64 + tag: default-runner:linux-ubuntu-22.04-amd64-main-e4de304 diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 2cd86491..a3b8f74a 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -5,4 +5,5 @@ resources: - garm-operator_v1alpha1_organization.yaml - garm-operator_v1alpha1_image.yaml - garm-operator_v1alpha1_repository.yaml + - garm-operator_v1alpha1_runner.yaml #+kubebuilder:scaffold:manifestskustomizesamples diff --git a/go.mod b/go.mod index 9df815ed..04e43dd1 100644 --- a/go.mod +++ b/go.mod @@ -13,9 +13,11 @@ require ( github.com/knadh/koanf/providers/file v0.1.0 github.com/knadh/koanf/providers/posflag v0.1.0 github.com/knadh/koanf/v2 v2.0.1 + github.com/life4/genesis v1.10.2 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.16.0 github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.8.4 go.uber.org/mock v0.2.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.27.2 @@ -75,6 +77,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect diff --git a/go.sum b/go.sum index 57acecd6..19f167eb 100644 --- a/go.sum +++ b/go.sum @@ -200,6 +200,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/life4/genesis v1.10.2 h1:zbnkCt8YehybmAaWe++ZnRvNLLbstst1vAO/zcJdpoA= +github.com/life4/genesis v1.10.2/go.mod h1:jhY+sEN403+0uE54fjVAdVCYY8SCIrKioAatOlVJoGo= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= @@ -207,6 +209,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= +github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= diff --git a/internal/controller/pool_controller.go b/internal/controller/pool_controller.go index d29ca762..f9960b9c 100644 --- a/internal/controller/pool_controller.go +++ b/internal/controller/pool_controller.go @@ -33,7 +33,8 @@ import ( garmClient "github.com/mercedes-benz/garm-operator/pkg/client" "github.com/mercedes-benz/garm-operator/pkg/client/key" "github.com/mercedes-benz/garm-operator/pkg/event" - "github.com/mercedes-benz/garm-operator/pkg/garmpool" + "github.com/mercedes-benz/garm-operator/pkg/filter" + poolfilter "github.com/mercedes-benz/garm-operator/pkg/filter/pool" "github.com/mercedes-benz/garm-operator/pkg/util/annotations" ) @@ -62,7 +63,7 @@ func (r *PoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. return ctrl.Result{}, nil } log.Error(err, "cannot fetch Pool") - return ctrl.Result{}, err + return r.handleUpdateError(ctx, pool, err) } // Ignore objects that are paused @@ -78,7 +79,7 @@ func (r *PoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. Password: r.Password, }) if err != nil { - return ctrl.Result{}, err + return r.handleUpdateError(ctx, pool, err) } instanceClient := garmClient.NewInstanceClient() @@ -88,7 +89,7 @@ func (r *PoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. Password: r.Password, }) if err != nil { - return ctrl.Result{}, err + return r.handleUpdateError(ctx, pool, err) } // handle deletion @@ -444,11 +445,11 @@ func (r *PoolReconciler) getExistingGarmPoolBySpecs(ctx context.Context, garmCli if err != nil { return params.Pool{}, err } - filteredGarmPools := garmpool.Filter(garmPools.Payload, - garmpool.MatchesImage(image.Spec.Tag), - garmpool.MatchesFlavor(pool.Spec.Flavor), - garmpool.MatchesProvider(pool.Spec.ProviderName), - garmpool.MatchesGitHubScope(scope, id), + filteredGarmPools := filter.Match(garmPools.Payload, + poolfilter.MatchesImage(image.Spec.Tag), + poolfilter.MatchesFlavor(pool.Spec.Flavor), + poolfilter.MatchesProvider(pool.Spec.ProviderName), + poolfilter.MatchesGitHubScope(scope, id), ) if len(filteredGarmPools) > 1 { diff --git a/internal/controller/pool_controller_test.go b/internal/controller/pool_controller_test.go index e9fb9176..bc2c3aac 100644 --- a/internal/controller/pool_controller_test.go +++ b/internal/controller/pool_controller_test.go @@ -26,6 +26,8 @@ import ( "github.com/mercedes-benz/garm-operator/pkg/client/mock" ) +const namespaceName = "test-namespace" + func TestPoolController_ReconcileCreate(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() @@ -34,7 +36,6 @@ func TestPoolController_ReconcileCreate(t *testing.T) { poolID := "fb2bceeb-f74d-435d-9648-626c75cb23ce" enterpriseID := "93068607-2d0d-4b76-a950-0e40d31955b8" enterpriseName := "test-enterprise" - namespaceName := "test-namespace" tests := []struct { name string @@ -651,7 +652,6 @@ func TestPoolController_ReconcileDelete(t *testing.T) { poolID := "fb2bceeb-f74d-435d-9648-626c75cb23ce" enterpriseID := "93068607-2d0d-4b76-a950-0e40d31955b8" enterpriseName := "test-enterprise" - namespaceName := "test-namespace" tests := []struct { name string diff --git a/internal/controller/runner_controller.go b/internal/controller/runner_controller.go new file mode 100644 index 00000000..8c63c453 --- /dev/null +++ b/internal/controller/runner_controller.go @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: MIT + +package controller + +import ( + "context" + "strings" + "time" + + "github.com/cloudbase/garm/client/instances" + "github.com/cloudbase/garm/params" + "github.com/life4/genesis/slices" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/source" + + garmoperatorv1alpha1 "github.com/mercedes-benz/garm-operator/api/v1alpha1" + garmClient "github.com/mercedes-benz/garm-operator/pkg/client" + "github.com/mercedes-benz/garm-operator/pkg/client/key" + "github.com/mercedes-benz/garm-operator/pkg/config" + "github.com/mercedes-benz/garm-operator/pkg/filter" + instancefilter "github.com/mercedes-benz/garm-operator/pkg/filter/instance" +) + +// RunnerReconciler reconciles a Runner object +type RunnerReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + + BaseURL string + Username string + Password string +} + +//+kubebuilder:rbac:groups=garm-operator.mercedes-benz.com,namespace=xxxxx,resources=runners,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=garm-operator.mercedes-benz.com,namespace=xxxxx,resources=runners/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=garm-operator.mercedes-benz.com,namespace=xxxxx,resources=runners/finalizers,verbs=update + +func (r *RunnerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := log.FromContext(ctx) + log.Info("Reconciling runners...", "Request", req) + + instanceClient, err := r.instanceClient() + if err != nil { + return ctrl.Result{}, err + } + + return r.reconcile(ctx, req, instanceClient) +} + +func (r *RunnerReconciler) reconcile(ctx context.Context, req ctrl.Request, instanceClient garmClient.InstanceClient) (ctrl.Result, error) { + // try fetch runner instance in garm db with events coming from reconcile loop events of RunnerCR or from manually enqueued events of garm api. + garmRunner, err := r.getGarmRunnerInstance(instanceClient, req.Name) + if err != nil { + return ctrl.Result{}, err + } + + // only create RunnerCR if it does not yet exist + runner := &garmoperatorv1alpha1.Runner{} + if err := r.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: strings.ToLower(req.Name)}, runner); err != nil { + return r.handleCreateRunnerCR(ctx, req, err, garmRunner) + } + + // delete runner in garm db + if !runner.ObjectMeta.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, instanceClient, garmRunner) + } + + // sync garm runner status back to RunnerCR + return r.updateRunnerStatus(ctx, runner, garmRunner) +} + +func (r *RunnerReconciler) handleCreateRunnerCR(ctx context.Context, req ctrl.Request, fetchErr error, garmRunner *params.Instance) (ctrl.Result, error) { + if apierrors.IsNotFound(fetchErr) && garmRunner != nil { + return r.createRunnerCR(ctx, garmRunner, req.Namespace) + } + return ctrl.Result{}, fetchErr +} + +func (r *RunnerReconciler) createRunnerCR(ctx context.Context, garmRunner *params.Instance, namespace string) (ctrl.Result, error) { + log := log.FromContext(ctx) + log.Info("Creating Runner", "Runner", garmRunner.Name) + + runnerObj := &garmoperatorv1alpha1.Runner{ + ObjectMeta: metav1.ObjectMeta{ + Name: strings.ToLower(garmRunner.Name), + Namespace: namespace, + }, + Spec: garmoperatorv1alpha1.RunnerSpec{}, + } + err := r.Create(ctx, runnerObj) + if err != nil { + return ctrl.Result{}, err + } + + if err := r.ensureFinalizer(ctx, runnerObj); err != nil { + return ctrl.Result{}, err + } + + if _, err := r.updateRunnerStatus(ctx, runnerObj, garmRunner); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +func (r *RunnerReconciler) reconcileDelete(ctx context.Context, runnerClient garmClient.InstanceClient, garmRunner *params.Instance) (ctrl.Result, error) { + log := log.FromContext(ctx) + if garmRunner != nil { + log.Info("Deleting Runner in Garm", "Runner Name", garmRunner.Name) + err := runnerClient.DeleteInstance(instances.NewDeleteInstanceParams().WithInstanceName(garmRunner.Name)) + if err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil +} + +func (r *RunnerReconciler) getGarmRunnerInstance(client garmClient.InstanceClient, name string) (*params.Instance, error) { + allInstances, err := client.ListInstances(instances.NewListInstancesParams().WithDefaults()) + if err != nil { + return nil, err + } + + filteredInstances := filter.Match(allInstances.Payload, instancefilter.MatchesName(name)) + if len(filteredInstances) == 0 { + return nil, nil + } + + return &filteredInstances[0], nil +} + +func (r *RunnerReconciler) ensureFinalizer(ctx context.Context, runner *garmoperatorv1alpha1.Runner) error { + if !controllerutil.ContainsFinalizer(runner, key.RunnerFinalizerName) { + controllerutil.AddFinalizer(runner, key.RunnerFinalizerName) + return r.Update(ctx, runner) + } + return nil +} + +func (r *RunnerReconciler) updateRunnerStatus(ctx context.Context, runner *garmoperatorv1alpha1.Runner, garmRunner *params.Instance) (ctrl.Result, error) { + if garmRunner == nil { + return ctrl.Result{}, nil + } + + log := log.FromContext(ctx) + log.Info("Update runner status...") + + poolName := garmRunner.PoolID + pools := &garmoperatorv1alpha1.PoolList{} + err := r.List(ctx, pools) + if err == nil { + pools.FilterByFields(garmoperatorv1alpha1.MatchesID(garmRunner.PoolID)) + + if len(pools.Items) > 0 { + poolName = pools.Items[0].Name + } + } + + runner.Status.ID = garmRunner.ID + runner.Status.ProviderID = garmRunner.ProviderID + runner.Status.AgentID = garmRunner.AgentID + runner.Status.Name = garmRunner.Name + runner.Status.OSType = garmRunner.OSType + runner.Status.OSName = garmRunner.OSName + runner.Status.OSVersion = garmRunner.OSVersion + runner.Status.OSArch = garmRunner.OSArch + runner.Status.Addresses = garmRunner.Addresses + runner.Status.Status = garmRunner.Status + runner.Status.InstanceStatus = garmRunner.RunnerStatus + runner.Status.PoolID = poolName + runner.Status.ProviderFault = string(garmRunner.ProviderFault) + runner.Status.GitHubRunnerGroup = garmRunner.GitHubRunnerGroup + + if err := r.Status().Update(ctx, runner); err != nil { + log.Error(err, "unable to update Runner status") + return ctrl.Result{}, err + } + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *RunnerReconciler) SetupWithManager(mgr ctrl.Manager, eventChan chan event.GenericEvent) error { + c, err := ctrl.NewControllerManagedBy(mgr). + For(&garmoperatorv1alpha1.Runner{}). + Build(r) + if err != nil { + return err + } + + return c.Watch(&source.Channel{Source: eventChan}, &handler.EnqueueRequestForObject{}) +} + +func (r *RunnerReconciler) PollRunnerInstances(ctx context.Context, eventChan chan event.GenericEvent) { + log := log.FromContext(ctx) + ticker := time.NewTicker(5 * time.Second) + for { + select { + case <-ctx.Done(): + log.Info("Closing event channel for runners...") + close(eventChan) + return + case _ = <-ticker.C: + log.Info("Polling Runners...") + instanceClient, err := r.instanceClient() + if err != nil { + log.Error(err, "Failed to create InstanceClient") + } + + err = r.EnqueueRunnerInstances(ctx, instanceClient, eventChan) + if err != nil { + log.Error(err, "Failed polling runner instances") + } + } + } +} + +func (r *RunnerReconciler) EnqueueRunnerInstances(ctx context.Context, instanceClient garmClient.InstanceClient, eventChan chan event.GenericEvent) error { + pools, err := r.fetchPools(ctx) + if err != nil { + return err + } + + if len(pools.Items) < 1 { + return nil + } + + // fetching runners by pools to ensure only runners belonging to pools in same namespace are being shown + garmRunnerInstances, err := r.fetchRunnerInstancesByNamespacedPools(instanceClient, pools) + if err != nil { + return err + } + + // compares garm db with RunnerCRs and deletes RunnerCRs not present in garm db + err = r.cleanUpNotMatchingRunnerCRs(ctx, garmRunnerInstances) + if err != nil { + return err + } + + // triggers controller to reconcile based on instances in garm db + enqeueRunnerEvents(garmRunnerInstances, eventChan) + return nil +} + +func enqeueRunnerEvents(garmRunnerInstances params.Instances, eventChan chan event.GenericEvent) { + for _, runner := range garmRunnerInstances { + runnerObj := garmoperatorv1alpha1.Runner{ + ObjectMeta: metav1.ObjectMeta{ + Name: strings.ToLower(runner.Name), + Namespace: config.Config.Operator.WatchNamespace, + }, + } + + e := event.GenericEvent{ + Object: &runnerObj, + } + + eventChan <- e + } +} + +func (r *RunnerReconciler) cleanUpNotMatchingRunnerCRs(ctx context.Context, garmRunnerInstances params.Instances) error { + runnerCRList := &garmoperatorv1alpha1.RunnerList{} + err := r.List(ctx, runnerCRList) + if err != nil { + return err + } + + runnerCRNameList := slices.Map(runnerCRList.Items, func(runner garmoperatorv1alpha1.Runner) string { + return runner.Name + }) + + runnerInstanceNameList := slices.Map(garmRunnerInstances, func(runner params.Instance) string { + return strings.ToLower(runner.Name) + }) + + runnersToDelete := getRunnerDiff(runnerCRNameList, runnerInstanceNameList) + log.Log.V(1).Info("Deleting runners: ", "Runners", runnersToDelete) + + for _, runnerName := range runnersToDelete { + runner := &garmoperatorv1alpha1.Runner{} + err := r.Get(ctx, types.NamespacedName{Namespace: config.Config.Operator.WatchNamespace, Name: runnerName}, runner) + if err != nil { + return err + } + + if runner.DeletionTimestamp.IsZero() { + err = r.Delete(ctx, runner) + if err != nil { + return err + } + } + + // getting RunnerCR from cache again before removing finalizer, as in the meantime object has changed + err = r.Get(ctx, types.NamespacedName{Namespace: config.Config.Operator.WatchNamespace, Name: runnerName}, runner) + if err != nil { + return err + } + + if controllerutil.ContainsFinalizer(runner, key.RunnerFinalizerName) { + controllerutil.RemoveFinalizer(runner, key.RunnerFinalizerName) + if err := r.Update(ctx, runner); err != nil { + return err + } + } + } + return nil +} + +func (r *RunnerReconciler) fetchPools(ctx context.Context) (*garmoperatorv1alpha1.PoolList, error) { + pools := &garmoperatorv1alpha1.PoolList{} + err := r.List(ctx, pools) + if err != nil { + return nil, err + } + return pools, nil +} + +func (r *RunnerReconciler) fetchRunnerInstancesByNamespacedPools(instanceClient garmClient.InstanceClient, pools *garmoperatorv1alpha1.PoolList) (params.Instances, error) { + garmRunnerInstances := params.Instances{} + for _, p := range pools.Items { + if p.Status.ID == "" { + continue + } + poolRunners, err := instanceClient.ListPoolInstances(instances.NewListPoolInstancesParams().WithPoolID(p.Status.ID)) + if err != nil { + return nil, err + } + garmRunnerInstances = append(garmRunnerInstances, poolRunners.Payload...) + } + return garmRunnerInstances, nil +} + +func (r *RunnerReconciler) instanceClient() (garmClient.InstanceClient, error) { + instanceClient := garmClient.NewInstanceClient() + err := instanceClient.Login(garmClient.GarmScopeParams{ + BaseURL: r.BaseURL, + Username: r.Username, + Password: r.Password, + }) + return instanceClient, err +} + +func getRunnerDiff(runnerCRs, garmRunners []string) []string { + var diff []string + + for _, runnerCR := range runnerCRs { + if !slices.Contains(garmRunners, runnerCR) { + diff = append(diff, runnerCR) + } + } + return diff +} diff --git a/internal/controller/runner_controller_test.go b/internal/controller/runner_controller_test.go new file mode 100644 index 00000000..58f92917 --- /dev/null +++ b/internal/controller/runner_controller_test.go @@ -0,0 +1,475 @@ +// SPDX-License-Identifier: MIT +package controller + +import ( + "context" + "reflect" + "strings" + "testing" + "time" + + commonParams "github.com/cloudbase/garm-provider-common/params" + "github.com/cloudbase/garm/client/instances" + "github.com/cloudbase/garm/params" + "github.com/life4/genesis/slices" + "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/event" + + garmoperatorv1alpha1 "github.com/mercedes-benz/garm-operator/api/v1alpha1" + "github.com/mercedes-benz/garm-operator/pkg/client/key" + "github.com/mercedes-benz/garm-operator/pkg/client/mock" + "github.com/mercedes-benz/garm-operator/pkg/config" +) + +func TestRunnerReconciler_reconcileCreate(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + tests := []struct { + name string + req ctrl.Request + runtimeObjects []runtime.Object + expectGarmRequest func(m *mock.MockInstanceClientMockRecorder) + wantErr bool + expectedObject *garmoperatorv1alpha1.Runner + }{ + { + name: "Create Runner CR", + req: ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "road-runner-k8s-FY5snJcv5dzn", + Namespace: "runner", + }, + }, + runtimeObjects: []runtime.Object{}, + expectGarmRequest: func(m *mock.MockInstanceClientMockRecorder) { + response := params.Instances{ + params.Instance{ + Name: "road-runner-k8s-FY5snJcv5dzn", + AgentID: 120, + ID: "8215f6c6-486e-4893-84df-3231b185a148", + OSArch: "amd64", + OSType: "linux", + PoolID: "a46553c6-ad87-454b-b5f5-a1c468d78c1e", + ProviderID: "kubernetes_external", + Status: commonParams.InstanceRunning, + RunnerStatus: params.RunnerIdle, + }, + } + + m.ListInstances(instances.NewListInstancesParams()).Return(&instances.ListInstancesOK{Payload: response}, nil) + }, + wantErr: false, + expectedObject: &garmoperatorv1alpha1.Runner{ + TypeMeta: metav1.TypeMeta{ + Kind: "Runner", + APIVersion: garmoperatorv1alpha1.GroupVersion.Group + "/" + garmoperatorv1alpha1.GroupVersion.Version, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "road-runner-k8s-fy5snjcv5dzn", + Namespace: "runner", + Finalizers: []string{ + key.RunnerFinalizerName, + }, + }, + Spec: garmoperatorv1alpha1.RunnerSpec{}, + Status: garmoperatorv1alpha1.RunnerStatus{ + Name: "road-runner-k8s-FY5snJcv5dzn", + AgentID: 120, + ID: "8215f6c6-486e-4893-84df-3231b185a148", + OSArch: "amd64", + OSType: "linux", + PoolID: "a46553c6-ad87-454b-b5f5-a1c468d78c1e", + ProviderID: "kubernetes_external", + Status: commonParams.InstanceRunning, + InstanceStatus: params.RunnerIdle, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + schemeBuilder := runtime.SchemeBuilder{ + garmoperatorv1alpha1.AddToScheme, + } + + err := schemeBuilder.AddToScheme(scheme.Scheme) + if err != nil { + t.Fatal(err) + } + var runtimeObjects []runtime.Object + runtimeObjects = append(runtimeObjects, tt.runtimeObjects...) + client := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(runtimeObjects...).WithStatusSubresource(&garmoperatorv1alpha1.Runner{}).Build() + + // create a fake reconciler + reconciler := &RunnerReconciler{ + Client: client, + BaseURL: "http://domain.does.not.exist:9997", + Username: "admin", + Password: "admin", + Recorder: record.NewFakeRecorder(3), + } + + mockInstanceClient := mock.NewMockInstanceClient(mockCtrl) + tt.expectGarmRequest(mockInstanceClient.EXPECT()) + + _, err = reconciler.reconcile(context.Background(), tt.req, mockInstanceClient) + if (err != nil) != tt.wantErr { + t.Errorf("RunnerReconciler.reconcile() error = %v, wantErr %v", err, tt.wantErr) + return + } + + runner := &garmoperatorv1alpha1.Runner{} + err = client.Get(context.Background(), types.NamespacedName{Namespace: tt.req.Namespace, Name: strings.ToLower(tt.req.Name)}, runner) + if (err != nil) != tt.wantErr { + t.Errorf("RunnerReconciler.reconcile() error = %v, wantErr %v", err, tt.wantErr) + return + } + + // empty resource version to avoid comparison errors + runner.ObjectMeta.ResourceVersion = "" + if !reflect.DeepEqual(runner, tt.expectedObject) { + t.Errorf("RunnerReconciler.reconcile() got = %#v, want %#v", runner, tt.expectedObject) + } + }) + } +} + +func TestRunnerReconciler_reconcileDeleteGarmRunner(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + now := metav1.NewTime(time.Now().UTC()) + + tests := []struct { + name string + req ctrl.Request + runtimeObjects []runtime.Object + expectGarmRequest func(m *mock.MockInstanceClientMockRecorder) + wantErr bool + expectedObject *garmoperatorv1alpha1.Runner + }{ + { + name: "Delete Runner in Garm DB, when Runner CR is marked with deletion timestamp", + req: ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "road-runner-k8s-fy5snjcv5dzn", + Namespace: "runner", + }, + }, + runtimeObjects: []runtime.Object{ + &garmoperatorv1alpha1.Runner{ + TypeMeta: metav1.TypeMeta{ + Kind: "Runner", + APIVersion: garmoperatorv1alpha1.GroupVersion.Group + "/" + garmoperatorv1alpha1.GroupVersion.Version, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "road-runner-k8s-fy5snjcv5dzn", + Namespace: "runner", + Finalizers: []string{ + key.RunnerFinalizerName, + }, + DeletionTimestamp: &now, + }, + Spec: garmoperatorv1alpha1.RunnerSpec{}, + Status: garmoperatorv1alpha1.RunnerStatus{ + Name: "road-runner-k8s-FY5snJcv5dzn", + AgentID: 120, + ID: "8215f6c6-486e-4893-84df-3231b185a148", + OSArch: "amd64", + OSType: "linux", + PoolID: "a46553c6-ad87-454b-b5f5-a1c468d78c1e", + ProviderID: "kubernetes_external", + Status: commonParams.InstanceRunning, + InstanceStatus: params.RunnerIdle, + }, + }, + }, + expectGarmRequest: func(m *mock.MockInstanceClientMockRecorder) { + response := params.Instances{ + params.Instance{ + Name: "road-runner-k8s-FY5snJcv5dzn", + AgentID: 120, + ID: "8215f6c6-486e-4893-84df-3231b185a148", + OSArch: "amd64", + OSType: "linux", + PoolID: "a46553c6-ad87-454b-b5f5-a1c468d78c1e", + ProviderID: "kubernetes_external", + Status: commonParams.InstanceRunning, + RunnerStatus: params.RunnerIdle, + }, + } + + m.ListInstances(instances.NewListInstancesParams()).Return(&instances.ListInstancesOK{Payload: response}, nil) + + m.DeleteInstance(instances.NewDeleteInstanceParams().WithInstanceName("road-runner-k8s-FY5snJcv5dzn")).Return(nil) + }, + wantErr: false, + expectedObject: &garmoperatorv1alpha1.Runner{ + TypeMeta: metav1.TypeMeta{ + Kind: "Runner", + APIVersion: garmoperatorv1alpha1.GroupVersion.Group + "/" + garmoperatorv1alpha1.GroupVersion.Version, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "road-runner-k8s-fy5snjcv5dzn", + Namespace: "runner", + Finalizers: []string{ + key.RunnerFinalizerName, + }, + DeletionTimestamp: &now, + }, + Spec: garmoperatorv1alpha1.RunnerSpec{}, + Status: garmoperatorv1alpha1.RunnerStatus{ + Name: "road-runner-k8s-FY5snJcv5dzn", + AgentID: 120, + ID: "8215f6c6-486e-4893-84df-3231b185a148", + OSArch: "amd64", + OSType: "linux", + PoolID: "a46553c6-ad87-454b-b5f5-a1c468d78c1e", + ProviderID: "kubernetes_external", + Status: commonParams.InstanceRunning, + InstanceStatus: params.RunnerIdle, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + schemeBuilder := runtime.SchemeBuilder{ + garmoperatorv1alpha1.AddToScheme, + } + + err := schemeBuilder.AddToScheme(scheme.Scheme) + if err != nil { + t.Fatal(err) + } + var runtimeObjects []runtime.Object + runtimeObjects = append(runtimeObjects, tt.runtimeObjects...) + client := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(runtimeObjects...).WithStatusSubresource(&garmoperatorv1alpha1.Runner{}).Build() + + // create a fake reconciler + reconciler := &RunnerReconciler{ + Client: client, + BaseURL: "http://domain.does.not.exist:9997", + Username: "admin", + Password: "admin", + Recorder: record.NewFakeRecorder(3), + } + + mockInstanceClient := mock.NewMockInstanceClient(mockCtrl) + tt.expectGarmRequest(mockInstanceClient.EXPECT()) + + _, err = reconciler.reconcile(context.Background(), tt.req, mockInstanceClient) + if (err != nil) != tt.wantErr { + t.Errorf("RunnerReconciler.reconcile() error = %v, wantErr %v", err, tt.wantErr) + return + } + + runner := &garmoperatorv1alpha1.Runner{} + err = client.Get(context.Background(), types.NamespacedName{Namespace: tt.req.Namespace, Name: strings.ToLower(tt.req.Name)}, runner) + if (err != nil) != tt.wantErr { + t.Errorf("RunnerReconciler.reconcile() error = %v, wantErr %v", err, tt.wantErr) + return + } + + assert.NotZero(t, runner.ObjectMeta.DeletionTimestamp) + }) + } +} + +func TestRunnerReconciler_reconcileDeleteCR(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + tests := []struct { + name string + req ctrl.Request + runtimeObjects []runtime.Object + expectGarmRequest func(m *mock.MockInstanceClientMockRecorder) + wantErr bool + expectedEvents []event.GenericEvent + }{ + { + name: "Delete Runner CR with no matching entry in Garm DB", + req: ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "road-runner-k8s-fy5snjcv5dzn", + Namespace: "runner", + }, + }, + runtimeObjects: []runtime.Object{ + &garmoperatorv1alpha1.Runner{ + TypeMeta: metav1.TypeMeta{ + Kind: "Runner", + APIVersion: garmoperatorv1alpha1.GroupVersion.Group + "/" + garmoperatorv1alpha1.GroupVersion.Version, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "road-runner-k8s-fy5snjcv5dzn", + Namespace: "test-namespace", + Finalizers: []string{ + key.RunnerFinalizerName, + }, + }, + Spec: garmoperatorv1alpha1.RunnerSpec{}, + Status: garmoperatorv1alpha1.RunnerStatus{ + Name: "road-runner-k8s-FY5snJcv5dzn", + AgentID: 120, + ID: "8215f6c6-486e-4893-84df-3231b185a148", + OSArch: "amd64", + OSType: "linux", + PoolID: "a46553c6-ad87-454b-b5f5-a1c468d78c1e", + ProviderID: "road-runner-k8s-fy5snjcv5dzn", + Status: commonParams.InstanceRunning, + InstanceStatus: params.RunnerIdle, + }, + }, + &garmoperatorv1alpha1.Runner{ + TypeMeta: metav1.TypeMeta{ + Kind: "Runner", + APIVersion: garmoperatorv1alpha1.GroupVersion.Group + "/" + garmoperatorv1alpha1.GroupVersion.Version, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "road-runner-k8s-n6kq2mt3k4qr", + Namespace: "test-namespace", + Finalizers: []string{ + key.RunnerFinalizerName, + }, + }, + Spec: garmoperatorv1alpha1.RunnerSpec{}, + Status: garmoperatorv1alpha1.RunnerStatus{ + Name: "road-runner-k8s-n6KQ2Mt3k4qr", + AgentID: 130, + ID: "13d31cad-588b-4ea8-8015-052a76ad3dd3", + OSArch: "amd64", + OSType: "linux", + PoolID: "a46553c6-ad87-454b-b5f5-a1c468d78c1e", + ProviderID: "road-runner-k8s-n6kq2mt3k4qr", + Status: commonParams.InstanceRunning, + InstanceStatus: params.RunnerIdle, + }, + }, + &garmoperatorv1alpha1.Pool{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pool", + APIVersion: garmoperatorv1alpha1.GroupVersion.Group + "/" + garmoperatorv1alpha1.GroupVersion.Version, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-enterprise-pool", + Namespace: "test-namespace", + }, + Spec: garmoperatorv1alpha1.PoolSpec{ + GitHubScopeRef: corev1.TypedLocalObjectReference{ + APIGroup: &garmoperatorv1alpha1.GroupVersion.Group, + Kind: string(garmoperatorv1alpha1.EnterpriseScope), + Name: "my-enterprise", + }, + ProviderName: "kubernetes_external", + MaxRunners: 5, + MinIdleRunners: 3, + ImageName: "ubuntu-image", + Flavor: "medium", + OSType: "linux", + OSArch: "arm64", + Tags: []string{"kubernetes", "linux", "arm64", "ubuntu"}, + Enabled: true, + RunnerBootstrapTimeout: 20, + ExtraSpecs: "", + GitHubRunnerGroup: "", + }, + Status: garmoperatorv1alpha1.PoolStatus{ + ID: "a46553c6-ad87-454b-b5f5-a1c468d78c1e", + }, + }, + }, + expectGarmRequest: func(m *mock.MockInstanceClientMockRecorder) { + response := &instances.ListPoolInstancesOK{Payload: params.Instances{ + params.Instance{ + Name: "road-runner-k8s-FY5snJcv5dzn", + AgentID: 120, + ID: "8215f6c6-486e-4893-84df-3231b185a148", + OSArch: "amd64", + OSType: "linux", + PoolID: "a46553c6-ad87-454b-b5f5-a1c468d78c1e", + ProviderID: "road-runner-k8s-fy5snjcv5dzn", + Status: commonParams.InstanceRunning, + RunnerStatus: params.RunnerIdle, + }, + }} + m.ListPoolInstances(instances.NewListPoolInstancesParams().WithPoolID("a46553c6-ad87-454b-b5f5-a1c468d78c1e")).Return(response, nil) + }, + wantErr: false, + expectedEvents: []event.GenericEvent{ + { + Object: &garmoperatorv1alpha1.Runner{ + ObjectMeta: metav1.ObjectMeta{ + Name: "road-runner-k8s-fy5snjcv5dzn", + Namespace: "test-namespace", + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + schemeBuilder := runtime.SchemeBuilder{ + garmoperatorv1alpha1.AddToScheme, + } + + err := schemeBuilder.AddToScheme(scheme.Scheme) + if err != nil { + t.Fatal(err) + } + var runtimeObjects []runtime.Object + runtimeObjects = append(runtimeObjects, tt.runtimeObjects...) + client := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(runtimeObjects...).WithStatusSubresource(&garmoperatorv1alpha1.Runner{}).Build() + + // create a fake reconciler + reconciler := &RunnerReconciler{ + Client: client, + BaseURL: "http://domain.does.not.exist:9997", + Username: "admin", + Password: "admin", + Recorder: record.NewFakeRecorder(3), + } + + mockInstanceClient := mock.NewMockInstanceClient(mockCtrl) + tt.expectGarmRequest(mockInstanceClient.EXPECT()) + + config.Config.Operator.WatchNamespace = "test-namespace" + fakeChan := make(chan event.GenericEvent) + + go func() { + err = reconciler.EnqueueRunnerInstances(context.Background(), mockInstanceClient, fakeChan) + if (err != nil) != tt.wantErr { + t.Errorf("RunnerReconciler.EnqueueRunnerInstances() error = %v, wantErr %v", err, tt.wantErr) + return + } + close(fakeChan) + }() + + var eventCount int + for obj := range fakeChan { + t.Logf("Received Event: %s", obj.Object.GetName()) + filtered := slices.Filter(tt.expectedEvents, func(e event.GenericEvent) bool { + return e.Object.GetName() == obj.Object.GetName() && e.Object.GetNamespace() == obj.Object.GetNamespace() + }) + eventCount++ + assert.Equal(t, 1, len(filtered)) + } + assert.Equal(t, len(tt.expectedEvents), eventCount) + }) + } +} diff --git a/pkg/client/client.go b/pkg/client/client.go index 9f587907..57c9a9a4 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -119,3 +119,11 @@ func initializeGarm(ctx context.Context, garmParams GarmScopeParams) error { return nil } + +func IsNotFoundError(err interface{}) bool { + apiErr, ok := err.(runtime.ClientResponseStatus) + if !ok { + return false + } + return apiErr.IsCode(404) +} diff --git a/pkg/client/key/key.go b/pkg/client/key/key.go index f5976171..1725ce0e 100644 --- a/pkg/client/key/key.go +++ b/pkg/client/key/key.go @@ -8,5 +8,6 @@ const ( OrganizationFinalizerName = groupName + "/organization" RepositoryFinalizerName = groupName + "/repository" PoolFinalizerName = groupName + "/pool" + RunnerFinalizerName = groupName + "/runner" PausedAnnotation = groupName + "/paused" ) diff --git a/pkg/filter/filter.go b/pkg/filter/filter.go new file mode 100644 index 00000000..c21852d8 --- /dev/null +++ b/pkg/filter/filter.go @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: MIT + +package filter + +type Predicate[T any] func(T) bool + +func Match[T any](items []T, predicates ...Predicate[T]) []T { + var filteredPools []T + + for _, pool := range items { + match := true + for _, predicate := range predicates { + if !predicate(pool) { + match = false + break + } + } + if match { + filteredPools = append(filteredPools, pool) + } + } + + return filteredPools +} diff --git a/pkg/filter/instance/instance.go b/pkg/filter/instance/instance.go new file mode 100644 index 00000000..397cf014 --- /dev/null +++ b/pkg/filter/instance/instance.go @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: MIT + +package instance + +import ( + "strings" + + "github.com/cloudbase/garm/params" + + "github.com/mercedes-benz/garm-operator/pkg/filter" +) + +func MatchesName(name string) filter.Predicate[params.Instance] { + return func(i params.Instance) bool { + return strings.EqualFold(i.Name, name) + } +} diff --git a/pkg/garmpool/filter.go b/pkg/filter/pool/pool.go similarity index 55% rename from pkg/garmpool/filter.go rename to pkg/filter/pool/pool.go index f65e1168..94b3ab08 100644 --- a/pkg/garmpool/filter.go +++ b/pkg/filter/pool/pool.go @@ -1,16 +1,15 @@ // SPDX-License-Identifier: MIT -package garmpool +package pool import ( "github.com/cloudbase/garm/params" garmoperatorv1alpha1 "github.com/mercedes-benz/garm-operator/api/v1alpha1" + "github.com/mercedes-benz/garm-operator/pkg/filter" ) -type Predicate func(params.Pool) bool - -func MatchesGitHubScope(scope garmoperatorv1alpha1.GitHubScopeKind, id string) Predicate { +func MatchesGitHubScope(scope garmoperatorv1alpha1.GitHubScopeKind, id string) filter.Predicate[params.Pool] { return func(p params.Pool) bool { if scope == garmoperatorv1alpha1.EnterpriseScope { return p.EnterpriseID == id @@ -27,39 +26,20 @@ func MatchesGitHubScope(scope garmoperatorv1alpha1.GitHubScopeKind, id string) P } } -func MatchesImage(image string) Predicate { +func MatchesImage(image string) filter.Predicate[params.Pool] { return func(p params.Pool) bool { return p.Image == image } } -func MatchesFlavor(flavor string) Predicate { +func MatchesFlavor(flavor string) filter.Predicate[params.Pool] { return func(p params.Pool) bool { return p.Flavor == flavor } } -func MatchesProvider(provider string) Predicate { +func MatchesProvider(provider string) filter.Predicate[params.Pool] { return func(p params.Pool) bool { return p.ProviderName == provider } } - -func Filter(pools []params.Pool, predicates ...func(pool params.Pool) bool) []params.Pool { - var filteredPools []params.Pool - - for _, pool := range pools { - match := true - for _, predicate := range predicates { - if !predicate(pool) { - match = false - break - } - } - if match { - filteredPools = append(filteredPools, pool) - } - } - - return filteredPools -}