diff --git a/Makefile b/Makefile index 5f5530ab9a6..47ed1f2d340 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,9 @@ fmt: vet: go vet ./pkg/... ./cmd/... +manifests: + go run vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go all + # Generate code generate: ifndef GOPATH diff --git a/OWNERS b/OWNERS index c3627a3f2df..1050753d551 100644 --- a/OWNERS +++ b/OWNERS @@ -5,6 +5,7 @@ approvers: - richardsliu - hougangliu - johnugeorge + - anchovYu reviewers: - libbyandhelen - texasmichelle diff --git a/cmd/katib-controller/v1alpha2/main.go b/cmd/katib-controller/v1alpha2/main.go index 2d8fdb154e7..9d2c89f6d96 100644 --- a/cmd/katib-controller/v1alpha2/main.go +++ b/cmd/katib-controller/v1alpha2/main.go @@ -15,10 +15,7 @@ limitations under the License. */ /* - StudyJobController is a controller (operator) for StudyJob - StudyJobController create and watch workers and metricscollectors. - The workers and metricscollectors are generated from template defined ConfigMap. - The workers and metricscollectors are kubernetes object. The default object is a Job and CronJob. + Katib-controller is a controller (operator) for Experiments and Trials */ package main @@ -30,10 +27,12 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/manager" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" "sigs.k8s.io/controller-runtime/pkg/runtime/signals" ) func main() { + logf.SetLogger(logf.ZapLogger(false)) // Get a config to talk to the apiserver cfg, err := config.GetConfig() if err != nil { @@ -41,7 +40,7 @@ func main() { log.Fatal(err) } - // Create a new StudyJobController to provide shared dependencies and start components + // Create a new katib controller to provide shared dependencies and start components mgr, err := manager.New(cfg, manager.Options{}) if err != nil { log.Printf("manager.New") @@ -56,7 +55,7 @@ func main() { log.Fatal(err) } - // Setup StudyJobController + // Setup katib controller if err := controller.AddToManager(mgr); err != nil { log.Printf("controller.AddToManager(mgr)") log.Fatal(err) @@ -64,6 +63,6 @@ func main() { log.Printf("Starting the Cmd.") - // Starting the StudyJobController + // Starting the katib controller log.Fatal(mgr.Start(signals.SetupSignalHandler())) } diff --git a/cmd/katib-controller/v1alpha3/main.go b/cmd/katib-controller/v1alpha3/main.go new file mode 100644 index 00000000000..fd0c51a7c17 --- /dev/null +++ b/cmd/katib-controller/v1alpha3/main.go @@ -0,0 +1,68 @@ +/* +Copyright 2018 The Kubeflow Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* + Katib-controller is a controller (operator) for Experiments and Trials +*/ +package main + +import ( + "log" + + "github.com/kubeflow/katib/pkg/api/operators/apis" + controller "github.com/kubeflow/katib/pkg/controller/v1alpha3" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/manager" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/runtime/signals" +) + +func main() { + logf.SetLogger(logf.ZapLogger(false)) + // Get a config to talk to the apiserver + cfg, err := config.GetConfig() + if err != nil { + log.Printf("config.GetConfig()") + log.Fatal(err) + } + + // Create a new katib controller to provide shared dependencies and start components + mgr, err := manager.New(cfg, manager.Options{}) + if err != nil { + log.Printf("manager.New") + log.Fatal(err) + } + + log.Printf("Registering Components.") + + // Setup Scheme for all resources + if err := apis.AddToScheme(mgr.GetScheme()); err != nil { + log.Printf("apis.AddToScheme") + log.Fatal(err) + } + + // Setup katib controller + if err := controller.AddToManager(mgr); err != nil { + log.Printf("controller.AddToManager(mgr)") + log.Fatal(err) + } + + log.Printf("Starting the Cmd.") + + // Starting the katib controller + log.Fatal(mgr.Start(signals.SetupSignalHandler())) +} diff --git a/manifests/v1alpha1/vizier/ui/deployment.yaml b/manifests/v1alpha1/vizier/ui/deployment.yaml index 913a06698f7..fb1fe1bcf73 100644 --- a/manifests/v1alpha1/vizier/ui/deployment.yaml +++ b/manifests/v1alpha1/vizier/ui/deployment.yaml @@ -23,6 +23,7 @@ spec: ports: - name: ui containerPort: 80 + serviceAccountName: katib-ui # resources: # requests: # cpu: 500m diff --git a/pkg/api/operators/apis/addtoscheme_experiments_v1alpha3.go b/pkg/api/operators/apis/addtoscheme_experiments_v1alpha3.go new file mode 100644 index 00000000000..cdaa6ec538a --- /dev/null +++ b/pkg/api/operators/apis/addtoscheme_experiments_v1alpha3.go @@ -0,0 +1,26 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apis + +import ( + "github.com/kubeflow/katib/pkg/api/operators/apis/experiment/v1alpha2" +) + +func init() { + // Register the types with the Scheme so the components can map objects to GroupVersionKinds and back + AddToSchemes = append(AddToSchemes, v1alpha2.SchemeBuilder.AddToScheme) +} diff --git a/pkg/api/operators/apis/addtoscheme_katib_v1alpha2.go b/pkg/api/operators/apis/addtoscheme_katib_v1alpha2.go new file mode 100644 index 00000000000..6bbd44cb0e8 --- /dev/null +++ b/pkg/api/operators/apis/addtoscheme_katib_v1alpha2.go @@ -0,0 +1,26 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apis + +import ( + experiments "github.com/kubeflow/katib/pkg/api/operators/apis/experiment/v1alpha2" + trials "github.com/kubeflow/katib/pkg/api/operators/apis/trial/v1alpha2" +) + +func init() { + // Register the types with the Scheme so the components can map objects to GroupVersionKinds and back + AddToSchemes = append(AddToSchemes, experiments.SchemeBuilder.AddToScheme, trials.SchemeBuilder.AddToScheme) +} diff --git a/pkg/api/operators/apis/addtoscheme_suggestion_v1alpha2.go b/pkg/api/operators/apis/addtoscheme_suggestion_v1alpha2.go new file mode 100644 index 00000000000..c565818f656 --- /dev/null +++ b/pkg/api/operators/apis/addtoscheme_suggestion_v1alpha2.go @@ -0,0 +1,26 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apis + +import ( + "github.com/kubeflow/katib/pkg/api/operators/apis/suggestion/v1alpha2" +) + +func init() { + // Register the types with the Scheme so the components can map objects to GroupVersionKinds and back + AddToSchemes = append(AddToSchemes, v1alpha2.SchemeBuilder.AddToScheme) +} diff --git a/pkg/api/operators/apis/addtoscheme_trial_v1alpha2.go b/pkg/api/operators/apis/addtoscheme_trial_v1alpha2.go new file mode 100644 index 00000000000..65be53d78f7 --- /dev/null +++ b/pkg/api/operators/apis/addtoscheme_trial_v1alpha2.go @@ -0,0 +1,26 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apis + +import ( + "github.com/kubeflow/katib/pkg/api/operators/apis/trial/v1alpha2" +) + +func init() { + // Register the types with the Scheme so the components can map objects to GroupVersionKinds and back + AddToSchemes = append(AddToSchemes, v1alpha2.SchemeBuilder.AddToScheme) +} diff --git a/pkg/api/operators/apis/experiment/v1alpha2/constants.go b/pkg/api/operators/apis/experiment/v1alpha2/constants.go new file mode 100644 index 00000000000..2a00316a5ce --- /dev/null +++ b/pkg/api/operators/apis/experiment/v1alpha2/constants.go @@ -0,0 +1,30 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package v1alpha2 + +const ( + // Default value of Spec.ParallelTrialCount + DefaultTrialParallelCount = 3 + + // Default value of Spec.ConfigMapName + DefaultTrialConfigMapName = "trial-template" + + // Default env name of katib namespace + DefaultKatibNamespaceEnvName = "KATIB_CORE_NAMESPACE" + + // Default value of Spec.TemplatePath + DefaultTrialTemplatePath = "defaultTrialTemplate.yaml" +) diff --git a/pkg/api/operators/apis/experiment/v1alpha2/doc.go b/pkg/api/operators/apis/experiment/v1alpha2/doc.go index 898eaf7ca19..5534c91a13d 100644 --- a/pkg/api/operators/apis/experiment/v1alpha2/doc.go +++ b/pkg/api/operators/apis/experiment/v1alpha2/doc.go @@ -18,5 +18,6 @@ limitations under the License. // +k8s:deepcopy-gen=package,register // +k8s:conversion-gen=github.com/kubeflow/katib/pkg/api/operators/apis/experiment/v1alpha2 // +k8s:defaulter-gen=TypeMeta +// +kubebuilder:subresource:status // +groupName=experiment.kubeflow.org package v1alpha2 diff --git a/pkg/api/operators/apis/experiment/v1alpha2/experiment_types.go b/pkg/api/operators/apis/experiment/v1alpha2/experiment_types.go index ffe20e9341e..dfdc125c4b3 100644 --- a/pkg/api/operators/apis/experiment/v1alpha2/experiment_types.go +++ b/pkg/api/operators/apis/experiment/v1alpha2/experiment_types.go @@ -35,10 +35,14 @@ type ExperimentSpec struct { TrialTemplate *TrialTemplate `json:"trialTemplate,omitempty"` // How many trials can be processed in parallel. - ParallelTrialCount int `json:"parallelTrialCount,omitempty"` + // Defaults to 3 + ParallelTrialCount *int `json:"parallelTrialCount,omitempty"` - // Total number of trials to run. - MaxTrialCount int `json:"maxTrialCount,omitempty"` + // Max completed trials to mark experiment as succeeded + MaxTrialCount *int `json:"maxTrialCount,omitempty"` + + // Max failed trials to mark experiment as failed. + MaxFailedTrialCount *int `json:"maxFailedTrialCount,omitempty"` // Whether to retain historical data in DB after deletion. RetainHistoricalData bool `json:"retainHistoricalData,omitempty"` @@ -75,8 +79,8 @@ type ExperimentStatus struct { // Current optimal trial parameters and observations. CurrentOptimalTrial OptimalTrial `json:"currentOptimalTrial,omitempty"` - // How many trials have successfully completed. - TrialsCompleted int `json:"trialsCompleted,omitempty"` + // How many trials have succeeded. + TrialsSucceeded int `json:"trialsSucceeded,omitempty"` // How many trials have failed. TrialsFailed int `json:"trialsFailed,omitempty"` @@ -86,6 +90,9 @@ type ExperimentStatus struct { // How many trials are currently pending. TrialsPending int `json:"trialsPending,omitempty"` + + // How many trials are currently running. + TrialsRunning int `json:"trialsRunning,omitempty"` } type OptimalTrial struct { @@ -154,7 +161,7 @@ type FeasibleSpace struct { type ObjectiveSpec struct { Type ObjectiveType `json:"type,omitempty"` - Goal float64 `json:"goal,omitempty"` + Goal *float64 `json:"goal,omitempty"` ObjectiveMetricName string `json:"objectiveMetricName,omitempty"` // This can be empty if we only care about the objective metric. // Note: If we adopt a push instead of pull mechanism, this can be omitted completely. @@ -206,6 +213,7 @@ type GoTemplate struct { // Structure of the Experiment custom resource. // +k8s:openapi-gen=true +// +kubebuilder:subresource:status type Experiment struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -231,7 +239,7 @@ type NasConfig struct { // GraphConfig contains a config of DAG type GraphConfig struct { - NumLayers int32 `json:"numLayers,omitempty"` + NumLayers *int32 `json:"numLayers,omitempty"` InputSizes []int32 `json:"inputSizes,omitempty"` OutputSizes []int32 `json:"outputSizes,omitempty"` } @@ -242,7 +250,6 @@ type Operation struct { Parameters []ParameterSpec `json:"parameterconfigs,omitempty"` } -// TODO - enable this during API implementation. -//func init() { -// SchemeBuilder.Register(&Experiment{}, &ExperimentList{}) -//} +func init() { + SchemeBuilder.Register(&Experiment{}, &ExperimentList{}) +} diff --git a/pkg/api/operators/apis/experiment/v1alpha2/register.go b/pkg/api/operators/apis/experiment/v1alpha2/register.go new file mode 100644 index 00000000000..f195fcc3e99 --- /dev/null +++ b/pkg/api/operators/apis/experiment/v1alpha2/register.go @@ -0,0 +1,42 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha2 contains API Schema definitions for the experiment v1alpha2 API group +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen=package,register +// +k8s:conversion-gen=github.com/kubeflow/katib/pkg/api/operators/apis/experiment/v1alpha2 +// +k8s:defaulter-gen=TypeMeta +// +kubebuilder:subresource:status +// +groupName=experiments.kubeflow.org +package v1alpha2 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/runtime/scheme" +) + +const ( + Group = "kubeflow.org" + Version = "v1alpha2" +) + +var ( + // SchemeGroupVersion is group version used to register these objects + SchemeGroupVersion = schema.GroupVersion{Group: Group, Version: Version} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/pkg/api/operators/apis/experiment/v1alpha2/util.go b/pkg/api/operators/apis/experiment/v1alpha2/util.go new file mode 100644 index 00000000000..bc9a1440a09 --- /dev/null +++ b/pkg/api/operators/apis/experiment/v1alpha2/util.go @@ -0,0 +1,122 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func getCondition(exp *Experiment, condType ExperimentConditionType) *ExperimentCondition { + for _, condition := range exp.Status.Conditions { + if condition.Type == condType { + return &condition + } + } + return nil +} + +func hasCondition(exp *Experiment, condType ExperimentConditionType) bool { + cond := getCondition(exp, condType) + if cond != nil && cond.Status == v1.ConditionTrue { + return true + } + return false +} + +func (exp *Experiment) removeCondition(condType ExperimentConditionType) { + var newConditions []ExperimentCondition + for _, c := range exp.Status.Conditions { + + if c.Type == condType { + continue + } + + newConditions = append(newConditions, c) + } + exp.Status.Conditions = newConditions +} + +func newCondition(conditionType ExperimentConditionType, status v1.ConditionStatus, reason, message string) ExperimentCondition { + return ExperimentCondition{ + Type: conditionType, + Status: status, + LastUpdateTime: metav1.Now(), + LastTransitionTime: metav1.Now(), + Reason: reason, + Message: message, + } +} + +func (exp *Experiment) IsCreated() bool { + return hasCondition(exp, ExperimentCreated) +} + +func (exp *Experiment) IsSucceeded() bool { + return hasCondition(exp, ExperimentSucceeded) +} + +func (exp *Experiment) IsFailed() bool { + return hasCondition(exp, ExperimentFailed) +} + +func (exp *Experiment) IsCompleted() bool { + return exp.IsSucceeded() || exp.IsFailed() +} + +func (exp *Experiment) setCondition(conditionType ExperimentConditionType, status v1.ConditionStatus, reason, message string) { + + newCond := newCondition(conditionType, status, reason, message) + currentCond := getCondition(exp, conditionType) + // Do nothing if condition doesn't change + if currentCond != nil && currentCond.Status == newCond.Status && currentCond.Reason == newCond.Reason { + return + } + + // Do not update lastTransitionTime if the status of the condition doesn't change. + if currentCond != nil && currentCond.Status == newCond.Status { + newCond.LastTransitionTime = currentCond.LastTransitionTime + } + + exp.removeCondition(conditionType) + exp.Status.Conditions = append(exp.Status.Conditions, newCond) +} + +func (exp *Experiment) MarkExperimentStatusCreated(reason, message string) { + exp.setCondition(ExperimentCreated, v1.ConditionTrue, reason, message) +} + +func (exp *Experiment) MarkExperimentStatusRunning(reason, message string) { + //exp.removeCondition(ExperimentRestarting) + exp.setCondition(ExperimentRunning, v1.ConditionTrue, reason, message) +} + +func (exp *Experiment) MarkExperimentStatusSucceeded(reason, message string) { + currentCond := getCondition(exp, ExperimentRunning) + if currentCond != nil { + exp.setCondition(ExperimentRunning, v1.ConditionFalse, currentCond.Reason, currentCond.Message) + } + exp.setCondition(ExperimentSucceeded, v1.ConditionTrue, reason, message) + +} + +func (exp *Experiment) MarkExperimentStatusFailed(reason, message string) { + currentCond := getCondition(exp, ExperimentRunning) + if currentCond != nil { + exp.setCondition(ExperimentRunning, v1.ConditionFalse, currentCond.Reason, currentCond.Message) + } + exp.setCondition(ExperimentFailed, v1.ConditionTrue, reason, message) +} diff --git a/pkg/api/operators/apis/experiment/v1alpha2/zz_generated.deepcopy.go b/pkg/api/operators/apis/experiment/v1alpha2/zz_generated.deepcopy.go index 344aed137ce..e5f0943453b 100644 --- a/pkg/api/operators/apis/experiment/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/api/operators/apis/experiment/v1alpha2/zz_generated.deepcopy.go @@ -186,6 +186,26 @@ func (in *ExperimentSpec) DeepCopyInto(out *ExperimentSpec) { *out = new(TrialTemplate) (*in).DeepCopyInto(*out) } + if in.ParallelTrialCount != nil { + in, out := &in.ParallelTrialCount, &out.ParallelTrialCount + *out = new(int) + **out = **in + } + if in.MaxTrialCount != nil { + in, out := &in.MaxTrialCount, &out.MaxTrialCount + *out = new(int) + **out = **in + } + if in.MaxFailedTrialCount != nil { + in, out := &in.MaxFailedTrialCount, &out.MaxFailedTrialCount + *out = new(int) + **out = **in + } + if in.NasConfig != nil { + in, out := &in.NasConfig, &out.NasConfig + *out = new(NasConfig) + (*in).DeepCopyInto(*out) + } return } @@ -277,9 +297,69 @@ func (in *GoTemplate) DeepCopy() *GoTemplate { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GraphConfig) DeepCopyInto(out *GraphConfig) { + *out = *in + if in.NumLayers != nil { + in, out := &in.NumLayers, &out.NumLayers + *out = new(int32) + **out = **in + } + if in.InputSizes != nil { + in, out := &in.InputSizes, &out.InputSizes + *out = make([]int32, len(*in)) + copy(*out, *in) + } + if in.OutputSizes != nil { + in, out := &in.OutputSizes, &out.OutputSizes + *out = make([]int32, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GraphConfig. +func (in *GraphConfig) DeepCopy() *GraphConfig { + if in == nil { + return nil + } + out := new(GraphConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NasConfig) DeepCopyInto(out *NasConfig) { + *out = *in + in.GraphConfig.DeepCopyInto(&out.GraphConfig) + if in.Operations != nil { + in, out := &in.Operations, &out.Operations + *out = make([]Operation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NasConfig. +func (in *NasConfig) DeepCopy() *NasConfig { + if in == nil { + return nil + } + out := new(NasConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectiveSpec) DeepCopyInto(out *ObjectiveSpec) { *out = *in + if in.Goal != nil { + in, out := &in.Goal, &out.Goal + *out = new(float64) + **out = **in + } if in.AdditionalMetricsNames != nil { in, out := &in.AdditionalMetricsNames, &out.AdditionalMetricsNames *out = make([]string, len(*in)) @@ -298,6 +378,29 @@ func (in *ObjectiveSpec) DeepCopy() *ObjectiveSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Operation) DeepCopyInto(out *Operation) { + *out = *in + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make([]ParameterSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Operation. +func (in *Operation) DeepCopy() *Operation { + if in == nil { + return nil + } + out := new(Operation) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OptimalTrial) DeepCopyInto(out *OptimalTrial) { *out = *in diff --git a/pkg/api/operators/apis/suggestion/group.go b/pkg/api/operators/apis/suggestion/group.go new file mode 100644 index 00000000000..5f68eff0d6d --- /dev/null +++ b/pkg/api/operators/apis/suggestion/group.go @@ -0,0 +1,18 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package suggestion contains suggestions API versions +package suggestion diff --git a/pkg/api/operators/apis/suggestion/v1alpha2/doc.go b/pkg/api/operators/apis/suggestion/v1alpha2/doc.go new file mode 100644 index 00000000000..f5aec3f5683 --- /dev/null +++ b/pkg/api/operators/apis/suggestion/v1alpha2/doc.go @@ -0,0 +1,23 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha2 contains API Schema definitions for the suggestions v1alpha2 API group +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen=package,register +// +k8s:conversion-gen=github.com/kubeflow/katib/pkg/apis/suggestions +// +k8s:defaulter-gen=TypeMeta +// +groupName=suggestions.kubeflow.org +package v1alpha2 diff --git a/pkg/api/operators/apis/suggestion/v1alpha2/register.go b/pkg/api/operators/apis/suggestion/v1alpha2/register.go new file mode 100644 index 00000000000..acd212ef654 --- /dev/null +++ b/pkg/api/operators/apis/suggestion/v1alpha2/register.go @@ -0,0 +1,46 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// NOTE: Boilerplate only. Ignore this file. + +// Package v1alpha2 contains API Schema definitions for the suggestions v1alpha2 API group +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen=package,register +// +k8s:conversion-gen=github.com/kubeflow/katib/pkg/apis/suggestions +// +k8s:defaulter-gen=TypeMeta +// +groupName=suggestions.kubeflow.org +package v1alpha2 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/runtime/scheme" +) + +var ( + // SchemeGroupVersion is group version used to register these objects + SchemeGroupVersion = schema.GroupVersion{Group: "suggestions.kubeflow.org", Version: "v1alpha2"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} + + // AddToScheme is required by pkg/client/... + AddToScheme = SchemeBuilder.AddToScheme +) + +// Resource is required by pkg/client/listers/... +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} diff --git a/pkg/api/operators/apis/suggestion/v1alpha2/suggestion_types.go b/pkg/api/operators/apis/suggestion/v1alpha2/suggestion_types.go new file mode 100644 index 00000000000..4e818429817 --- /dev/null +++ b/pkg/api/operators/apis/suggestion/v1alpha2/suggestion_types.go @@ -0,0 +1,125 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// SuggestionSpec defines the desired state of Suggestion +type SuggestionSpec struct { + // Number of desired pods. This is a pointer to distinguish between explicit + // zero and not specified. Defaults to 1. + // +optional + Replicas *int32 `json:"replicas,omitempty"` + + // Type of the suggestion. + Type SuggestionType `json:"type,omitempty"` + + // Defines if the suggestion needs previous results. + NeedHistory bool `json:"needHistory,omitempty"` + + // Template describes the pods that will be created. + Template v1.PodTemplateSpec `json:"template"` +} + +type SuggestionType string + +const ( + SuggestionTypeNAS = "NAS" + SuggestionTypeEarlyStopping = "EarlyStopping" + SuggestionTypeHPTuning = "HyperParameter" +) + +// SuggestionStatus defines the observed state of Suggestion +type SuggestionStatus struct { + // Represents time when the Experiment was acknowledged by the Experiment controller. + // It is not guaranteed to be set in happens-before order across separate operations. + // It is represented in RFC3339 form and is in UTC. + StartTime *metav1.Time `json:"startTime,omitempty"` + + // Represents time when the Experiment was completed. It is not guaranteed to + // be set in happens-before order across separate operations. + // It is represented in RFC3339 form and is in UTC. + CompletionTime *metav1.Time `json:"completionTime,omitempty"` + + // Represents last time when the Experiment was reconciled. It is not guaranteed to + // be set in happens-before order across separate operations. + // It is represented in RFC3339 form and is in UTC. + LastReconcileTime *metav1.Time `json:"lastReconcileTime,omitempty"` + + // List of observed runtime conditions for this Experiment. + Conditions []SuggestionCondition `json:"conditions,omitempty"` +} + +// +k8s:deepcopy-gen=true +// SuggestionCondition describes the state of the suggestion at a certain point. +type SuggestionCondition struct { + // Type of experiment condition. + Type SuggestionConditionType `json:"type"` + + // Status of the condition, one of True, False, Unknown. + Status v1.ConditionStatus `json:"status"` + + // The reason for the condition's last transition. + Reason string `json:"reason,omitempty"` + + // A human readable message indicating details about the transition. + Message string `json:"message,omitempty"` + + // The last time this condition was updated. + LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"` + + // Last time the condition transitioned from one status to another. + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` +} + +// SuggestionConditionType defines the state of an Suggestion. +type SuggestionConditionType string + +const ( + SuggestionDeploymentAvailable SuggestionConditionType = "DeploymentAvailable" + SuggestionDeploymentProgressing SuggestionConditionType = "DeploymentProgressing" + SuggestionDeploymentReplicaFailure SuggestionConditionType = "DeploymentReplicaFailure" +) + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// Suggestion is the Schema for the suggestions API +// +k8s:openapi-gen=true +type Suggestion struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec SuggestionSpec `json:"spec,omitempty"` + Status SuggestionStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// SuggestionList contains a list of Suggestion +type SuggestionList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Suggestion `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Suggestion{}, &SuggestionList{}) +} diff --git a/pkg/api/operators/apis/suggestion/v1alpha2/suggestion_types_test.go b/pkg/api/operators/apis/suggestion/v1alpha2/suggestion_types_test.go new file mode 100644 index 00000000000..d6f06cb0248 --- /dev/null +++ b/pkg/api/operators/apis/suggestion/v1alpha2/suggestion_types_test.go @@ -0,0 +1,58 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + "testing" + + "github.com/onsi/gomega" + "golang.org/x/net/context" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +func TestStorageSuggestion(t *testing.T) { + key := types.NamespacedName{ + Name: "foo", + Namespace: "default", + } + created := &Suggestion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "default", + }} + g := gomega.NewGomegaWithT(t) + + // Test Create + fetched := &Suggestion{} + g.Expect(c.Create(context.TODO(), created)).NotTo(gomega.HaveOccurred()) + + g.Expect(c.Get(context.TODO(), key, fetched)).NotTo(gomega.HaveOccurred()) + g.Expect(fetched).To(gomega.Equal(created)) + + // Test Updating the Labels + updated := fetched.DeepCopy() + updated.Labels = map[string]string{"hello": "world"} + g.Expect(c.Update(context.TODO(), updated)).NotTo(gomega.HaveOccurred()) + + g.Expect(c.Get(context.TODO(), key, fetched)).NotTo(gomega.HaveOccurred()) + g.Expect(fetched).To(gomega.Equal(updated)) + + // Test Delete + g.Expect(c.Delete(context.TODO(), fetched)).NotTo(gomega.HaveOccurred()) + g.Expect(c.Get(context.TODO(), key, fetched)).To(gomega.HaveOccurred()) +} diff --git a/pkg/api/operators/apis/suggestion/v1alpha2/v1alpha3_suite_test.go b/pkg/api/operators/apis/suggestion/v1alpha2/v1alpha3_suite_test.go new file mode 100644 index 00000000000..5733cafded3 --- /dev/null +++ b/pkg/api/operators/apis/suggestion/v1alpha2/v1alpha3_suite_test.go @@ -0,0 +1,55 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + "log" + "os" + "path/filepath" + "testing" + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +var cfg *rest.Config +var c client.Client + +func TestMain(m *testing.M) { + t := &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "..", "config", "crds")}, + } + + err := SchemeBuilder.AddToScheme(scheme.Scheme) + if err != nil { + log.Fatal(err) + } + + if cfg, err = t.Start(); err != nil { + log.Fatal(err) + } + + if c, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}); err != nil { + log.Fatal(err) + } + + code := m.Run() + t.Stop() + os.Exit(code) +} diff --git a/pkg/api/operators/apis/suggestion/v1alpha2/zz_generated.deepcopy.go b/pkg/api/operators/apis/suggestion/v1alpha2/zz_generated.deepcopy.go new file mode 100644 index 00000000000..7a0c2871033 --- /dev/null +++ b/pkg/api/operators/apis/suggestion/v1alpha2/zz_generated.deepcopy.go @@ -0,0 +1,160 @@ +// +build !ignore_autogenerated + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by main. DO NOT EDIT. + +package v1alpha2 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Suggestion) DeepCopyInto(out *Suggestion) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Suggestion. +func (in *Suggestion) DeepCopy() *Suggestion { + if in == nil { + return nil + } + out := new(Suggestion) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Suggestion) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SuggestionCondition) DeepCopyInto(out *SuggestionCondition) { + *out = *in + in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime) + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SuggestionCondition. +func (in *SuggestionCondition) DeepCopy() *SuggestionCondition { + if in == nil { + return nil + } + out := new(SuggestionCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SuggestionList) DeepCopyInto(out *SuggestionList) { + *out = *in + out.TypeMeta = in.TypeMeta + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Suggestion, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SuggestionList. +func (in *SuggestionList) DeepCopy() *SuggestionList { + if in == nil { + return nil + } + out := new(SuggestionList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *SuggestionList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SuggestionSpec) DeepCopyInto(out *SuggestionSpec) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + in.Template.DeepCopyInto(&out.Template) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SuggestionSpec. +func (in *SuggestionSpec) DeepCopy() *SuggestionSpec { + if in == nil { + return nil + } + out := new(SuggestionSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SuggestionStatus) DeepCopyInto(out *SuggestionStatus) { + *out = *in + if in.StartTime != nil { + in, out := &in.StartTime, &out.StartTime + *out = (*in).DeepCopy() + } + if in.CompletionTime != nil { + in, out := &in.CompletionTime, &out.CompletionTime + *out = (*in).DeepCopy() + } + if in.LastReconcileTime != nil { + in, out := &in.LastReconcileTime, &out.LastReconcileTime + *out = (*in).DeepCopy() + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]SuggestionCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SuggestionStatus. +func (in *SuggestionStatus) DeepCopy() *SuggestionStatus { + if in == nil { + return nil + } + out := new(SuggestionStatus) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/api/operators/apis/trial/v1alpha2/doc.go b/pkg/api/operators/apis/trial/v1alpha2/doc.go index 898eaf7ca19..e872d1f6505 100644 --- a/pkg/api/operators/apis/trial/v1alpha2/doc.go +++ b/pkg/api/operators/apis/trial/v1alpha2/doc.go @@ -13,10 +13,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package v1alpha2 contains API Schema definitions for the experiment v1alpha2 API group +// Package v1alpha2 contains API Schema definitions for the trial v1alpha2 API group // +k8s:openapi-gen=true // +k8s:deepcopy-gen=package,register -// +k8s:conversion-gen=github.com/kubeflow/katib/pkg/api/operators/apis/experiment/v1alpha2 +// +k8s:conversion-gen=github.com/kubeflow/katib/pkg/api/operators/apis/trial/v1alpha2 // +k8s:defaulter-gen=TypeMeta -// +groupName=experiment.kubeflow.org +// +kubebuilder:subresource:status +// +groupName=trial.kubeflow.org package v1alpha2 diff --git a/pkg/api/operators/apis/trial/v1alpha2/register.go b/pkg/api/operators/apis/trial/v1alpha2/register.go new file mode 100644 index 00000000000..2ae2627dc09 --- /dev/null +++ b/pkg/api/operators/apis/trial/v1alpha2/register.go @@ -0,0 +1,42 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha2 contains API Schema definitions for the trial v1alpha2 API group +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen=package,register +// +k8s:conversion-gen=github.com/kubeflow/katib/pkg/api/operators/apis/trial/v1alpha2 +// +k8s:defaulter-gen=TypeMeta +// +kubebuilder:subresource:status +// +groupName=trials.kubeflow.org +package v1alpha2 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/runtime/scheme" +) + +const ( + Group = "kubeflow.org" + Version = "v1alpha2" +) + +var ( + // SchemeGroupVersion is group version used to register these objects + SchemeGroupVersion = schema.GroupVersion{Group: Group, Version: Version} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/pkg/api/operators/apis/trial/v1alpha2/trial_types.go b/pkg/api/operators/apis/trial/v1alpha2/trial_types.go index b1bd2ca0d5f..1d07e1efabf 100644 --- a/pkg/api/operators/apis/trial/v1alpha2/trial_types.go +++ b/pkg/api/operators/apis/trial/v1alpha2/trial_types.go @@ -95,9 +95,9 @@ type TrialCondition struct { type TrialConditionType string const ( - TrialPending TrialConditionType = "Pending" + TrialCreated TrialConditionType = "Created" TrialRunning TrialConditionType = "Running" - TrialCompleted TrialConditionType = "Completed" + TrialSucceeded TrialConditionType = "Succeeded" TrialKilled TrialConditionType = "Killed" TrialFailed TrialConditionType = "Failed" ) @@ -107,6 +107,7 @@ const ( // Represents the structure of a Trial resource. // +k8s:openapi-gen=true +// +kubebuilder:subresource:status type Trial struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -124,7 +125,6 @@ type TrialList struct { Items []Trial `json:"items"` } -// TODO: Enable this later during API implementation. -//func init() { -// SchemeBuilder.Register(&Trial{}, &TrialList{}) -//} +func init() { + SchemeBuilder.Register(&Trial{}, &TrialList{}) +} diff --git a/pkg/api/operators/apis/trial/v1alpha2/util.go b/pkg/api/operators/apis/trial/v1alpha2/util.go new file mode 100644 index 00000000000..2fd19688dc3 --- /dev/null +++ b/pkg/api/operators/apis/trial/v1alpha2/util.go @@ -0,0 +1,137 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func getCondition(trial *Trial, condType TrialConditionType) *TrialCondition { + for _, condition := range trial.Status.Conditions { + if condition.Type == condType { + return &condition + } + } + return nil +} + +func hasCondition(trial *Trial, condType TrialConditionType) bool { + cond := getCondition(trial, condType) + if cond != nil && cond.Status == v1.ConditionTrue { + return true + } + return false +} + +func (trial *Trial) removeCondition(condType TrialConditionType) { + var newConditions []TrialCondition + for _, c := range trial.Status.Conditions { + + if c.Type == condType { + continue + } + + newConditions = append(newConditions, c) + } + trial.Status.Conditions = newConditions +} + +func newCondition(conditionType TrialConditionType, status v1.ConditionStatus, reason, message string) TrialCondition { + return TrialCondition{ + Type: conditionType, + Status: status, + LastUpdateTime: metav1.Now(), + LastTransitionTime: metav1.Now(), + Reason: reason, + Message: message, + } +} + +func (trial *Trial) IsCreated() bool { + return hasCondition(trial, TrialCreated) +} + +func (trial *Trial) IsRunning() bool { + return hasCondition(trial, TrialRunning) +} + +func (trial *Trial) IsSucceeded() bool { + return hasCondition(trial, TrialSucceeded) +} + +func (trial *Trial) IsFailed() bool { + return hasCondition(trial, TrialFailed) +} + +func (trial *Trial) IsKilled() bool { + return hasCondition(trial, TrialKilled) +} + +func (trial *Trial) IsCompleted() bool { + return trial.IsSucceeded() || trial.IsFailed() || trial.IsKilled() +} + +func (trial *Trial) setCondition(conditionType TrialConditionType, status v1.ConditionStatus, reason, message string) { + + newCond := newCondition(conditionType, status, reason, message) + currentCond := getCondition(trial, conditionType) + // Do nothing if condition doesn't change + if currentCond != nil && currentCond.Status == newCond.Status && currentCond.Reason == newCond.Reason { + return + } + + // Do not update lastTransitionTime if the status of the condition doesn't change. + if currentCond != nil && currentCond.Status == newCond.Status { + newCond.LastTransitionTime = currentCond.LastTransitionTime + } + + trial.removeCondition(conditionType) + trial.Status.Conditions = append(trial.Status.Conditions, newCond) +} + +func (trial *Trial) MarkTrialStatusCreated(reason, message string) { + trial.setCondition(TrialCreated, v1.ConditionTrue, reason, message) +} + +func (trial *Trial) MarkTrialStatusRunning(reason, message string) { + trial.setCondition(TrialRunning, v1.ConditionTrue, reason, message) +} + +func (trial *Trial) MarkTrialStatusSucceeded(reason, message string) { + currentCond := getCondition(trial, TrialRunning) + if currentCond != nil { + trial.setCondition(TrialRunning, v1.ConditionFalse, currentCond.Reason, currentCond.Message) + } + trial.setCondition(TrialSucceeded, v1.ConditionTrue, reason, message) + +} + +func (trial *Trial) MarkTrialStatusFailed(reason, message string) { + currentCond := getCondition(trial, TrialRunning) + if currentCond != nil { + trial.setCondition(TrialRunning, v1.ConditionFalse, currentCond.Reason, currentCond.Message) + } + trial.setCondition(TrialFailed, v1.ConditionTrue, reason, message) +} + +func (trial *Trial) MarkTrialStatusKilled(reason, message string) { + currentCond := getCondition(trial, TrialRunning) + if currentCond != nil { + trial.setCondition(TrialRunning, v1.ConditionFalse, currentCond.Reason, currentCond.Message) + } + trial.setCondition(TrialKilled, v1.ConditionTrue, reason, message) +} diff --git a/pkg/controller/v1alpha2/experiment/experiment_controller.go b/pkg/controller/v1alpha2/experiment/experiment_controller.go index db67dc60105..1ed1f477b2e 100644 --- a/pkg/controller/v1alpha2/experiment/experiment_controller.go +++ b/pkg/controller/v1alpha2/experiment/experiment_controller.go @@ -19,9 +19,11 @@ package experiment import ( "context" - experimentsv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/experiment/v1alpha2" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -29,9 +31,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" "sigs.k8s.io/controller-runtime/pkg/source" + + experimentsv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/experiment/v1alpha2" + trialsv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/trial/v1alpha2" + "github.com/kubeflow/katib/pkg/controller/v1alpha2/experiment/util" ) -var log = logf.Log.WithName("controller") +var log = logf.Log.WithName("experiment-controller") /** * USER ACTION REQUIRED: This is a scaffold file intended for the user to modify with their own Controller @@ -54,15 +60,31 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // Create a new controller c, err := controller.New("experiment-controller", mgr, controller.Options{Reconciler: r}) if err != nil { + log.Error(err, "Failed to create experiment controller") return err } // Watch for changes to Experiment err = c.Watch(&source.Kind{Type: &experimentsv1alpha2.Experiment{}}, &handler.EnqueueRequestForObject{}) if err != nil { + log.Error(err, "Experiment watch failed") return err } + // Watch for trials for the experiments + err = c.Watch( + &source.Kind{Type: &trialsv1alpha2.Trial{}}, + &handler.EnqueueRequestForOwner{ + IsController: true, + OwnerType: &experimentsv1alpha2.Experiment{}, + }) + + if err != nil { + log.Error(err, "Trial watch failed") + return err + } + + log.Info("Experiment controller created") return nil } @@ -80,8 +102,10 @@ type ReconcileExperiment struct { // +kubebuilder:rbac:groups=experiments.kubeflow.org,resources=experiments/status,verbs=get;update;patch func (r *ReconcileExperiment) Reconcile(request reconcile.Request) (reconcile.Result, error) { // Fetch the Experiment instance - instance := &experimentsv1alpha2.Experiment{} - err := r.Get(context.TODO(), request.NamespacedName, instance) + logger := log.WithValues("Experiment", request.NamespacedName) + original := &experimentsv1alpha2.Experiment{} + requeue := false + err := r.Get(context.TODO(), request.NamespacedName, original) if err != nil { if errors.IsNotFound(err) { // Object not found, return. Created objects are automatically garbage collected. @@ -89,8 +113,164 @@ func (r *ReconcileExperiment) Reconcile(request reconcile.Request) (reconcile.Re return reconcile.Result{}, nil } // Error reading the object - requeue the request. + logger.Error(err, "Experiment Get error") return reconcile.Result{}, err } + instance := original.DeepCopy() + + if instance.IsCompleted() { + + return reconcile.Result{}, nil + + } + if !instance.IsCreated() { + //Experiment not created in DB + err = util.CreateExperimentInDB(instance) + if err != nil { + logger.Error(err, "Create experiment in DB error") + return reconcile.Result{}, err + } + + if instance.Status.StartTime == nil { + now := metav1.Now() + instance.Status.StartTime = &now + } + msg := "Experiment is created" + instance.MarkExperimentStatusCreated(util.ExperimentCreatedReason, msg) + requeue = true + } else { + // Experiment already created in DB + err := r.ReconcileExperiment(instance) + if err != nil { + logger.Error(err, "Reconcile experiment error") + return reconcile.Result{}, err + } + } + + if !equality.Semantic.DeepEqual(original.Status, instance.Status) { + //assuming that only status change + err = util.UpdateExperimentStatusInDB(instance) + if err != nil { + logger.Error(err, "Update experiment status in DB error") + return reconcile.Result{}, err + } + err = r.Status().Update(context.TODO(), instance) + if err != nil { + logger.Error(err, "Update experiment instance status error") + return reconcile.Result{}, err + } + } + + return reconcile.Result{Requeue: requeue}, nil +} + +func (r *ReconcileExperiment) ReconcileExperiment(instance *experimentsv1alpha2.Experiment) error { + + logger := log.WithValues("Experiment", types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()}) + trials := &trialsv1alpha2.TrialList{} + labels := map[string]string{"experiment": instance.Name} + lo := &client.ListOptions{} + lo.MatchingLabels(labels).InNamespace(instance.Namespace) + + if err := r.List(context.TODO(), lo, trials); err != nil { + logger.Error(err, "Trial List error") + return err + } + if len(trials.Items) > 0 { + if err := util.UpdateExperimentStatus(instance, trials); err != nil { + logger.Error(err, "Update experiment status error") + return err + } + } + reconcileRequired := !instance.IsCompleted() + if reconcileRequired { + r.ReconcileTrials(instance) + } + return nil +} + +func (r *ReconcileExperiment) ReconcileTrials(instance *experimentsv1alpha2.Experiment) error { + + logger := log.WithValues("Experiment", types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()}) + parallelCount := 0 + + if instance.Spec.ParallelTrialCount != nil { + parallelCount = *instance.Spec.ParallelTrialCount + } else { + parallelCount = experimentsv1alpha2.DefaultTrialParallelCount + } + activeCount := instance.Status.TrialsPending + instance.Status.TrialsRunning + completedCount := instance.Status.TrialsSucceeded + instance.Status.TrialsFailed + instance.Status.TrialsKilled - return reconcile.Result{}, nil + if activeCount > parallelCount { + deleteCount := activeCount - parallelCount + if deleteCount > 0 { + //delete 'deleteCount' number of trails. Sort them? + logger.Info("DeleteTrials", "deleteCount", deleteCount) + if err := r.deleteTrials(instance, deleteCount); err != nil { + logger.Error(err, "Delete trials error") + return err + } + } + + } else if activeCount < parallelCount { + requiredActiveCount := 0 + if instance.Spec.MaxTrialCount == nil { + requiredActiveCount = parallelCount + } else { + requiredActiveCount = *instance.Spec.MaxTrialCount - completedCount + if requiredActiveCount > parallelCount { + requiredActiveCount = parallelCount + } + } + + addCount := requiredActiveCount - activeCount + if addCount < 0 { + logger.Info("Invalid setting", "requiredActiveCount", requiredActiveCount, "MaxTrialCount", + *instance.Spec.MaxTrialCount, "CompletedCount", completedCount) + addCount = 0 + } + + //create "addCount" number of trials + logger.Info("CreateTrials", "addCount", addCount) + if err := r.createTrials(instance, addCount); err != nil { + logger.Error(err, "Create trials error") + return err + } + + } + + return nil + +} + +func (r *ReconcileExperiment) createTrials(instance *experimentsv1alpha2.Experiment, addCount int) error { + + logger := log.WithValues("Experiment", types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()}) + trials, err := util.GetSuggestions(instance, addCount) + if err != nil { + logger.Error(err, "Get suggestions error") + return err + } + /*trials := []apiv1alpha2.Trial{ + apiv1alpha2.Trial{Spec: &apiv1alpha2.TrialSpec{}}, apiv1alpha2.Trial{Spec: &apiv1alpha2.TrialSpec{}}, + }*/ + + trialTemplate, err := r.getTrialTemplate(instance) + if err != nil { + logger.Error(err, "Get trial template error") + return err + } + for _, trial := range trials { + if err = r.createTrialInstance(instance, trial, trialTemplate); err != nil { + logger.Error(err, "Create trial instance error", "trial", trial) + continue + } + } + return nil +} + +func (r *ReconcileExperiment) deleteTrials(instance *experimentsv1alpha2.Experiment, deleteCount int) error { + + return nil } diff --git a/pkg/controller/v1alpha2/experiment/experiment_util.go b/pkg/controller/v1alpha2/experiment/experiment_util.go new file mode 100644 index 00000000000..cca21cf9030 --- /dev/null +++ b/pkg/controller/v1alpha2/experiment/experiment_util.go @@ -0,0 +1,123 @@ +package experiment + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "text/template" + + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + utilrand "k8s.io/apimachinery/pkg/util/rand" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + experimentsv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/experiment/v1alpha2" + trialsv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/trial/v1alpha2" + apiv1alpha2 "github.com/kubeflow/katib/pkg/api/v1alpha2" +) + +type TrialTemplateParams struct { + Experiment string + Trial string + NameSpace string + HyperParameters []*apiv1alpha2.ParameterAssignment +} + +func (r *ReconcileExperiment) createTrialInstance(expInstance *experimentsv1alpha2.Experiment, trialInstance *apiv1alpha2.Trial, trialTemplate *template.Template) error { + logger := log.WithValues("Experiment", types.NamespacedName{Name: expInstance.GetName(), Namespace: expInstance.GetNamespace()}) + + trial := &trialsv1alpha2.Trial{} + trial.Name = fmt.Sprintf("%s-%s", expInstance.GetName(), utilrand.String(8)) + trial.Namespace = expInstance.GetNamespace() + trial.Labels = map[string]string{"experiment": expInstance.GetName()} + + if err := controllerutil.SetControllerReference(expInstance, trial, r.scheme); err != nil { + logger.Error(err, "Set controller reference error") + return err + } + + trialParams := TrialTemplateParams{ + Experiment: expInstance.GetName(), + Trial: trial.Name, + NameSpace: trial.Namespace, + } + + var buf bytes.Buffer + if trialInstance.Spec != nil && trialInstance.Spec.ParameterAssignments != nil { + for _, p := range trialInstance.Spec.ParameterAssignments.Assignments { + trialParams.HyperParameters = append(trialParams.HyperParameters, p) + } + } + if err := trialTemplate.Execute(&buf, trialParams); err != nil { + logger.Error(err, "Template execute error") + return err + } + + trial.Spec.RunSpec = buf.String() + + if err := r.Create(context.TODO(), trial); err != nil { + logger.Error(err, "Trial create error", "Trial name", trial.Name) + return err + } + return nil + +} + +func (r *ReconcileExperiment) getTrialTemplate(instance *experimentsv1alpha2.Experiment) (*template.Template, error) { + + var err error + var tpl *template.Template = nil + logger := log.WithValues("Experiment", types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()}) + trialTemplate := instance.Spec.TrialTemplate + if trialTemplate != nil && trialTemplate.GoTemplate.RawTemplate != "" { + tpl, err = template.New("Trial").Parse(trialTemplate.GoTemplate.RawTemplate) + } else { + //default values if user hasn't set + configMapNS := os.Getenv(experimentsv1alpha2.DefaultKatibNamespaceEnvName) + configMapName := experimentsv1alpha2.DefaultTrialConfigMapName + templatePath := experimentsv1alpha2.DefaultTrialTemplatePath + + if trialTemplate != nil && trialTemplate.GoTemplate.TemplateSpec != nil { + templateSpec := trialTemplate.GoTemplate.TemplateSpec + if templateSpec.ConfigMapName != "" { + configMapName = templateSpec.ConfigMapName + } + if templateSpec.ConfigMapNamespace != "" { + configMapNS = templateSpec.ConfigMapNamespace + } + if templateSpec.TemplatePath != "" { + templatePath = templateSpec.TemplatePath + } + } + configMap, err := r.getConfigMap(configMapName, configMapNS) + if err != nil { + logger.Error(err, "Get config map error", "configMapName", configMapName, "configMapNS", configMapNS) + return nil, err + } + if configMapTemplate, ok := configMap[templatePath]; !ok { + err = errors.New(string(metav1.StatusReasonNotFound)) + logger.Error(err, "Config map template not found", "templatePath", templatePath) + return nil, err + } else { + tpl, err = template.New("Trial").Parse(configMapTemplate) + } + } + if err != nil { + logger.Error(err, "Template parse error") + return nil, err + } + + return tpl, nil +} + +func (r *ReconcileExperiment) getConfigMap(name, namespace string) (map[string]string, error) { + + configMap := &apiv1.ConfigMap{} + if err := r.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, configMap); err != nil { + return map[string]string{}, err + } + return configMap.Data, nil +} diff --git a/pkg/controller/v1alpha2/experiment/util/api_util.go b/pkg/controller/v1alpha2/experiment/util/api_util.go new file mode 100644 index 00000000000..f2b9a9b0cdc --- /dev/null +++ b/pkg/controller/v1alpha2/experiment/util/api_util.go @@ -0,0 +1,38 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + //v1 "k8s.io/api/core/v1" + + experimentsv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/experiment/v1alpha2" + trialapi "github.com/kubeflow/katib/pkg/api/v1alpha2" +) + +func CreateExperimentInDB(instance *experimentsv1alpha2.Experiment) error { + + return nil +} + +func UpdateExperimentStatusInDB(instance *experimentsv1alpha2.Experiment) error { + + return nil +} + +func GetSuggestions(instance *experimentsv1alpha2.Experiment, addCount int) ([]*trialapi.Trial, error) { + + return nil, nil +} diff --git a/pkg/controller/v1alpha2/experiment/util/status_util.go b/pkg/controller/v1alpha2/experiment/util/status_util.go new file mode 100644 index 00000000000..86c984413a9 --- /dev/null +++ b/pkg/controller/v1alpha2/experiment/util/status_util.go @@ -0,0 +1,154 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + + experimentsv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/experiment/v1alpha2" + trialsv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/trial/v1alpha2" +) + +var log = logf.Log.WithName("experiment-status-util") + +const ( + ExperimentCreatedReason = "ExperimentCreated" + ExperimentRunningReason = "ExperimentRunning" + ExperimentSucceededReason = "ExperimentSucceeded" + ExperimentFailedReason = "ExperimentFailed" + ExperimentKilledReason = "ExperimentKilled" +) + +func UpdateExperimentStatus(instance *experimentsv1alpha2.Experiment, trials *trialsv1alpha2.TrialList) error { + + isObjectiveGoalReached := updateTrialsSummary(instance, trials) + + updateExperimentStatusCondition(instance, isObjectiveGoalReached) + return nil + +} + +func updateTrialsSummary(instance *experimentsv1alpha2.Experiment, trials *trialsv1alpha2.TrialList) bool { + + var trialsPending, trialsRunning, trialsSucceeded, trialsFailed, trialsKilled int + var bestTrialValue float64 + bestTrialIndex := -1 + isObjectiveGoalReached := false + objectiveValueGoal := *instance.Spec.Objective.Goal + objectiveType := instance.Spec.Objective.Type + objectiveMetricName := instance.Spec.Objective.ObjectiveMetricName + + for index, trial := range trials.Items { + if trial.IsKilled() { + trialsKilled++ + } else if trial.IsFailed() { + trialsFailed++ + } else if trial.IsSucceeded() { + trialsSucceeded++ + } else if trial.IsRunning() { + trialsRunning++ + } else { + trialsPending++ + } + + objectiveMetricValue := getObjectiveMetricValue(trial, objectiveMetricName) + if objectiveMetricValue == nil { + log.Info("Objective metric name not found", "trial", trial.GetName()) + continue + } + + //intialize vars to objective metric value of the first trial + if bestTrialIndex == -1 { + bestTrialValue = *objectiveMetricValue + bestTrialIndex = index + } + + if objectiveType == experimentsv1alpha2.ObjectiveTypeMinimize { + if *objectiveMetricValue < bestTrialValue { + bestTrialValue = *objectiveMetricValue + bestTrialIndex = index + } + if bestTrialValue <= objectiveValueGoal { + isObjectiveGoalReached = true + } + } else if objectiveType == experimentsv1alpha2.ObjectiveTypeMaximize { + if *objectiveMetricValue > bestTrialValue { + bestTrialValue = *objectiveMetricValue + bestTrialIndex = index + } + if bestTrialValue >= objectiveValueGoal { + isObjectiveGoalReached = true + } + } + } + instance.Status.TrialsPending = trialsPending + instance.Status.TrialsRunning = trialsRunning + instance.Status.TrialsSucceeded = trialsSucceeded + instance.Status.TrialsFailed = trialsFailed + instance.Status.TrialsKilled = trialsKilled + + // if best trial is set + if bestTrialIndex != -1 { + bestTrial := trials.Items[bestTrialIndex] + + instance.Status.CurrentOptimalTrial.ParameterAssignments = []trialsv1alpha2.ParameterAssignment{} + for _, parameterAssigment := range bestTrial.Spec.ParameterAssignments { + instance.Status.CurrentOptimalTrial.ParameterAssignments = append(instance.Status.CurrentOptimalTrial.ParameterAssignments, parameterAssigment) + } + + instance.Status.CurrentOptimalTrial.Observation.Metrics = []trialsv1alpha2.Metric{} + for _, metric := range bestTrial.Status.Observation.Metrics { + instance.Status.CurrentOptimalTrial.Observation.Metrics = append(instance.Status.CurrentOptimalTrial.Observation.Metrics, metric) + } + } + return isObjectiveGoalReached +} + +func getObjectiveMetricValue(trial trialsv1alpha2.Trial, objectiveMetricName string) *float64 { + for _, metric := range trial.Status.Observation.Metrics { + if objectiveMetricName == metric.Name { + return &metric.Value + } + } + return nil +} + +func updateExperimentStatusCondition(instance *experimentsv1alpha2.Experiment, isObjectiveGoalReached bool) { + + completedTrialsCount := instance.Status.TrialsSucceeded + instance.Status.TrialsFailed + instance.Status.TrialsKilled + failedTrialsCount := instance.Status.TrialsFailed + + if isObjectiveGoalReached { + msg := "Experiment has succeeded because Objective goal has reached" + instance.MarkExperimentStatusSucceeded(ExperimentSucceededReason, msg) + return + } + + if (instance.Spec.MaxTrialCount != nil) && (completedTrialsCount >= *instance.Spec.MaxTrialCount) { + msg := "Experiment has succeeded because max trial count has reached" + instance.MarkExperimentStatusSucceeded(ExperimentSucceededReason, msg) + return + } + + if (instance.Spec.MaxFailedTrialCount != nil) && (failedTrialsCount >= *instance.Spec.MaxFailedTrialCount) { + msg := "Experiment has failed because max failed count has reached" + instance.MarkExperimentStatusFailed(ExperimentFailedReason, msg) + return + } + + msg := "Experiment is running" + instance.MarkExperimentStatusRunning(ExperimentRunningReason, msg) +} diff --git a/pkg/controller/v1alpha2/trial/metriccollector.go b/pkg/controller/v1alpha2/trial/metriccollector.go new file mode 100644 index 00000000000..7f5b24bbec0 --- /dev/null +++ b/pkg/controller/v1alpha2/trial/metriccollector.go @@ -0,0 +1,16 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package trial diff --git a/pkg/controller/v1alpha2/trial/trial_controller.go b/pkg/controller/v1alpha2/trial/trial_controller.go index 537c4836545..0167698ebff 100644 --- a/pkg/controller/v1alpha2/trial/trial_controller.go +++ b/pkg/controller/v1alpha2/trial/trial_controller.go @@ -17,21 +17,32 @@ limitations under the License. package trial import ( + "bytes" "context" - trialsv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/trial/v1alpha2" + batchv1 "k8s.io/api/batch/v1" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + k8syaml "k8s.io/apimachinery/pkg/util/yaml" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" "sigs.k8s.io/controller-runtime/pkg/source" + + trialsv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/trial/v1alpha2" + "github.com/kubeflow/katib/pkg/controller/v1alpha2/trial/util" ) -var log = logf.Log.WithName("controller") +var log = logf.Log.WithName("trial-controller") /** * USER ACTION REQUIRED: This is a scaffold file intended for the user to modify with their own Controller @@ -54,15 +65,28 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { // Create a new controller c, err := controller.New("trial-controller", mgr, controller.Options{Reconciler: r}) if err != nil { + log.Error(err, "Create trial controller error") return err } // Watch for changes to Trial err = c.Watch(&source.Kind{Type: &trialsv1alpha2.Trial{}}, &handler.EnqueueRequestForObject{}) if err != nil { + log.Error(err, "Trial watch error") + return err + } + + err = c.Watch(&source.Kind{Type: &batchv1.Job{}}, + &handler.EnqueueRequestForOwner{ + IsController: true, + OwnerType: &trialsv1alpha2.Trial{}, + }) + if err != nil { + log.Error(err, "Job watch error") return err } + log.Info("Trial controller created") return nil } @@ -80,8 +104,10 @@ type ReconcileTrial struct { // +kubebuilder:rbac:groups=trials.kubeflow.org,resources=trials/status,verbs=get;update;patch func (r *ReconcileTrial) Reconcile(request reconcile.Request) (reconcile.Result, error) { // Fetch the Trial instance - instance := &trialsv1alpha2.Trial{} - err := r.Get(context.TODO(), request.NamespacedName, instance) + logger := log.WithValues("Trial", request.NamespacedName) + original := &trialsv1alpha2.Trial{} + requeue := false + err := r.Get(context.TODO(), request.NamespacedName, original) if err != nil { if errors.IsNotFound(err) { // Object not found, return. Created objects are automatically garbage collected. @@ -89,8 +115,136 @@ func (r *ReconcileTrial) Reconcile(request reconcile.Request) (reconcile.Result, return reconcile.Result{}, nil } // Error reading the object - requeue the request. + logger.Error(err, "Trial Get error") return reconcile.Result{}, err } - return reconcile.Result{}, nil + instance := original.DeepCopy() + + if instance.IsCompleted() { + + return reconcile.Result{}, nil + + } + if !instance.IsCreated() { + //Trial not created in DB + err = util.CreateTrialInDB(instance) + if err != nil { + logger.Error(err, "Create trial in DB error") + return reconcile.Result{}, err + } + if instance.Status.StartTime == nil { + now := metav1.Now() + instance.Status.StartTime = &now + } + msg := "Trial is created" + instance.MarkTrialStatusCreated(util.TrialCreatedReason, msg) + requeue = true + + } else { + // Trial already created in DB + err := r.reconcileTrial(instance) + if err != nil { + logger.Error(err, "Reconcile trial error") + return reconcile.Result{}, err + } + } + + if !equality.Semantic.DeepEqual(original.Status, instance.Status) { + //assuming that only status change + err = util.UpdateTrialStatusInDB(instance) + if err != nil { + logger.Error(err, "Update trial status in DB error") + return reconcile.Result{}, err + } + err = r.Status().Update(context.TODO(), instance) + if err != nil { + logger.Error(err, "Update trial instance status error") + return reconcile.Result{}, err + } + } + + return reconcile.Result{Requeue: requeue}, nil +} + +func (r *ReconcileTrial) reconcileTrial(instance *trialsv1alpha2.Trial) error { + + var err error + logger := log.WithValues("Trial", types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()}) + desiredJob, err := r.getDesiredJobSpec(instance) + if err != nil { + logger.Error(err, "Job Spec Get error") + return err + } + + deployedJob, err := r.reconcileJob(instance, desiredJob) + if err != nil { + logger.Error(err, "Reconcile job error") + return err + } + + //Job already exists + //TODO Can desired Spec differ from deployedSpec? + if deployedJob != nil { + if err = util.UpdateTrialStatusCondition(instance, deployedJob); err != nil { + logger.Error(err, "Update trial status condition error") + return err + } + if err = util.UpdateTrialStatusObservation(instance, deployedJob); err != nil { + logger.Error(err, "Update trial status observation error") + return err + } + } + return nil +} + +func (r *ReconcileTrial) reconcileJob(instance *trialsv1alpha2.Trial, desiredJob *unstructured.Unstructured) (*unstructured.Unstructured, error) { + + var err error + logger := log.WithValues("Trial", types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()}) + apiVersion := desiredJob.GetAPIVersion() + kind := desiredJob.GetKind() + gvk := schema.FromAPIVersionAndKind(apiVersion, kind) + + deployedJob := &unstructured.Unstructured{} + deployedJob.SetGroupVersionKind(gvk) + err = r.Get(context.TODO(), types.NamespacedName{Name: desiredJob.GetName(), Namespace: desiredJob.GetNamespace()}, deployedJob) + if err != nil { + if errors.IsNotFound(err) { + logger.Info("Creating Job", "kind", kind) + err = r.Create(context.TODO(), desiredJob) + if err != nil { + logger.Error(err, "Create job error") + return nil, err + } + } else { + logger.Error(err, "Trial Get error") + return nil, err + } + } + + //TODO create Metric colletor + + msg := "Trial is running" + instance.MarkTrialStatusRunning(util.TrialRunningReason, msg) + return deployedJob, nil +} + +func (r *ReconcileTrial) getDesiredJobSpec(instance *trialsv1alpha2.Trial) (*unstructured.Unstructured, error) { + + bufSize := 1024 + logger := log.WithValues("Trial", types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()}) + buf := bytes.NewBufferString(instance.Spec.RunSpec) + + desiredJobSpec := &unstructured.Unstructured{} + if err := k8syaml.NewYAMLOrJSONDecoder(buf, bufSize).Decode(desiredJobSpec); err != nil { + logger.Error(err, "Yaml decode error") + return nil, err + } + if err := controllerutil.SetControllerReference(instance, desiredJobSpec, r.scheme); err != nil { + logger.Error(err, "Set controller reference error") + return nil, err + } + + return desiredJobSpec, nil } diff --git a/pkg/controller/v1alpha2/trial/util/api_util.go b/pkg/controller/v1alpha2/trial/util/api_util.go new file mode 100644 index 00000000000..93db52db3ed --- /dev/null +++ b/pkg/controller/v1alpha2/trial/util/api_util.go @@ -0,0 +1,37 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + //v1 "k8s.io/api/core/v1" + + trialsv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/trial/v1alpha2" +) + +func CreateTrialInDB(instance *trialsv1alpha2.Trial) error { + + return nil +} + +func UpdateTrialStatusInDB(instance *trialsv1alpha2.Trial) error { + + return nil +} + +func GetTrialObservation(instance *trialsv1alpha2.Trial) error { + + return nil +} diff --git a/pkg/controller/v1alpha2/trial/util/status_util.go b/pkg/controller/v1alpha2/trial/util/status_util.go new file mode 100644 index 00000000000..64688f922aa --- /dev/null +++ b/pkg/controller/v1alpha2/trial/util/status_util.go @@ -0,0 +1,94 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + //v1 "k8s.io/api/core/v1" + + batchv1 "k8s.io/api/batch/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + + trialsv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/trial/v1alpha2" + commonv1beta1 "github.com/kubeflow/tf-operator/pkg/apis/common/v1beta1" +) + +var log = logf.Log.WithName("trial-status-util") + +const ( + DefaultJobKind = "Job" + TrialCreatedReason = "TrialCreated" + TrialRunningReason = "TrialRunning" + TrialSucceededReason = "TrialSucceeded" + TrialFailedReason = "TrialFailed" + TrialKilledReason = "TrialKilled" +) + +func UpdateTrialStatusCondition(instance *trialsv1alpha2.Trial, deployedJob *unstructured.Unstructured) error { + + kind := deployedJob.GetKind() + status, ok, unerr := unstructured.NestedFieldCopy(deployedJob.Object, "status") + + if ok { + statusMap := status.(map[string]interface{}) + switch kind { + + case DefaultJobKind: + jobStatus := batchv1.JobStatus{} + err := runtime.DefaultUnstructuredConverter.FromUnstructured(statusMap, &jobStatus) + if err != nil { + log.Error(err, "Convert unstructured to status error") + return err + } + if jobStatus.Active == 0 && jobStatus.Succeeded > 0 { + msg := "Trial has succeeded" + instance.MarkTrialStatusSucceeded(TrialSucceededReason, msg) + } else if jobStatus.Failed > 0 { + msg := "Trial has failed" + instance.MarkTrialStatusFailed(TrialFailedReason, msg) + } + default: + jobStatus := commonv1beta1.JobStatus{} + err := runtime.DefaultUnstructuredConverter.FromUnstructured(statusMap, &jobStatus) + + if err != nil { + log.Error(err, "Convert unstructured to status error") + return err + } + if len(jobStatus.Conditions) > 0 { + lc := jobStatus.Conditions[len(jobStatus.Conditions)-1] + if lc.Type == commonv1beta1.JobSucceeded { + msg := "Trial has succeeded" + instance.MarkTrialStatusSucceeded(TrialSucceededReason, msg) + } else if lc.Type == commonv1beta1.JobFailed { + msg := "Trial has failed" + instance.MarkTrialStatusFailed(TrialFailedReason, msg) + } + } + } + } else if unerr != nil { + log.Error(unerr, "NestedFieldCopy unstructured to status error") + return unerr + } + return nil +} + +func UpdateTrialStatusObservation(instance *trialsv1alpha2.Trial, deployedJob *unstructured.Unstructured) error { + + // read GetObservationLog call and update observation field + return nil +} diff --git a/pkg/controller/v1alpha3/add_experiment.go b/pkg/controller/v1alpha3/add_experiment.go new file mode 100644 index 00000000000..0821470506d --- /dev/null +++ b/pkg/controller/v1alpha3/add_experiment.go @@ -0,0 +1,26 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "github.com/kubeflow/katib/pkg/controller/v1alpha3/experiment" +) + +func init() { + // AddToManagerFuncs is a list of functions to create controllers and add them to a manager. + AddToManagerFuncs = append(AddToManagerFuncs, experiment.Add) +} diff --git a/pkg/controller/v1alpha3/add_suggestion.go b/pkg/controller/v1alpha3/add_suggestion.go new file mode 100644 index 00000000000..63af969e84b --- /dev/null +++ b/pkg/controller/v1alpha3/add_suggestion.go @@ -0,0 +1,26 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "github.com/kubeflow/katib/pkg/controller/v1alpha3/suggestion" +) + +func init() { + // AddToManagerFuncs is a list of functions to create controllers and add them to a manager. + AddToManagerFuncs = append(AddToManagerFuncs, suggestion.Add) +} diff --git a/pkg/controller/v1alpha3/add_trial.go b/pkg/controller/v1alpha3/add_trial.go new file mode 100644 index 00000000000..229e8a7f6f4 --- /dev/null +++ b/pkg/controller/v1alpha3/add_trial.go @@ -0,0 +1,26 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "github.com/kubeflow/katib/pkg/controller/v1alpha3/trial" +) + +func init() { + // AddToManagerFuncs is a list of functions to create controllers and add them to a manager. + AddToManagerFuncs = append(AddToManagerFuncs, trial.Add) +} diff --git a/pkg/controller/v1alpha3/controller.go b/pkg/controller/v1alpha3/controller.go new file mode 100644 index 00000000000..b2acc50372b --- /dev/null +++ b/pkg/controller/v1alpha3/controller.go @@ -0,0 +1,33 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +// AddToManagerFuncs is a list of functions to add all Controllers to the Manager +var AddToManagerFuncs []func(manager.Manager) error + +// AddToManager adds all Controllers to the Manager +func AddToManager(m manager.Manager) error { + for _, f := range AddToManagerFuncs { + if err := f(m); err != nil { + return err + } + } + return nil +} diff --git a/pkg/controller/v1alpha3/experiment/experiment_controller.go b/pkg/controller/v1alpha3/experiment/experiment_controller.go new file mode 100644 index 00000000000..9ab463c8c38 --- /dev/null +++ b/pkg/controller/v1alpha3/experiment/experiment_controller.go @@ -0,0 +1,168 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package experiment + +import ( + "context" + "reflect" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/source" + + experimentsv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/experiment/v1alpha2" +) + +var log = logf.Log.WithName("controller") + +/** +* USER ACTION REQUIRED: This is a scaffold file intended for the user to modify with their own Controller +* business logic. Delete these comments after modifying this file.* + */ + +// Add creates a new Experiment Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(mgr manager.Manager) error { + return add(mgr, newReconciler(mgr)) +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager) reconcile.Reconciler { + return &ReconcileExperiment{Client: mgr.GetClient(), scheme: mgr.GetScheme()} +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New("experiment-controller", mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + + // Watch for changes to Experiment + err = c.Watch(&source.Kind{Type: &experimentsv1alpha2.Experiment{}}, &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } + + // TODO(user): Modify this to be the types you create + // Uncomment watch a Deployment created by Experiment - change this for objects you create + err = c.Watch(&source.Kind{Type: &appsv1.Deployment{}}, &handler.EnqueueRequestForOwner{ + IsController: true, + OwnerType: &experimentsv1alpha2.Experiment{}, + }) + if err != nil { + return err + } + + return nil +} + +var _ reconcile.Reconciler = &ReconcileExperiment{} + +// ReconcileExperiment reconciles a Experiment object +type ReconcileExperiment struct { + client.Client + scheme *runtime.Scheme +} + +// Reconcile reads that state of the cluster for a Experiment object and makes changes based on the state read +// and what is in the Experiment.Spec +// TODO(user): Modify this Reconcile function to implement your Controller logic. The scaffolding writes +// a Deployment as an example +// Automatically generate RBAC rules to allow the Controller to read and write Deployments +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=deployments/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=experiments.kubeflow.org,resources=experiments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=experiments.kubeflow.org,resources=experiments/status,verbs=get;update;patch +func (r *ReconcileExperiment) Reconcile(request reconcile.Request) (reconcile.Result, error) { + // Fetch the Experiment instance + instance := &experimentsv1alpha2.Experiment{} + err := r.Get(context.TODO(), request.NamespacedName, instance) + if err != nil { + if errors.IsNotFound(err) { + // Object not found, return. Created objects are automatically garbage collected. + // For additional cleanup logic use finalizers. + return reconcile.Result{}, nil + } + // Error reading the object - requeue the request. + return reconcile.Result{}, err + } + + // TODO(user): Change this to be the object type created by your controller + // Define the desired Deployment object + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name + "-deployment", + Namespace: instance.Namespace, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"deployment": instance.Name + "-deployment"}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"deployment": instance.Name + "-deployment"}}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, + } + if err := controllerutil.SetControllerReference(instance, deploy, r.scheme); err != nil { + return reconcile.Result{}, err + } + + // TODO(user): Change this for the object type created by your controller + // Check if the Deployment already exists + found := &appsv1.Deployment{} + err = r.Get(context.TODO(), types.NamespacedName{Name: deploy.Name, Namespace: deploy.Namespace}, found) + if err != nil && errors.IsNotFound(err) { + log.Info("Creating Deployment", "namespace", deploy.Namespace, "name", deploy.Name) + err = r.Create(context.TODO(), deploy) + return reconcile.Result{}, err + } else if err != nil { + return reconcile.Result{}, err + } + + // TODO(user): Change this for the object type created by your controller + // Update the found object and write the result back if there are any changes + if !reflect.DeepEqual(deploy.Spec, found.Spec) { + found.Spec = deploy.Spec + log.Info("Updating Deployment", "namespace", deploy.Namespace, "name", deploy.Name) + err = r.Update(context.TODO(), found) + if err != nil { + return reconcile.Result{}, err + } + } + return reconcile.Result{}, nil +} diff --git a/pkg/controller/v1alpha3/experiment/experiment_controller_suite_test.go b/pkg/controller/v1alpha3/experiment/experiment_controller_suite_test.go new file mode 100644 index 00000000000..f797fc76e04 --- /dev/null +++ b/pkg/controller/v1alpha3/experiment/experiment_controller_suite_test.go @@ -0,0 +1,75 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package experiment + +import ( + stdlog "log" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/kubeflow/katib/pkg/apis" + "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +var cfg *rest.Config + +func TestMain(m *testing.M) { + t := &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, + } + apis.AddToScheme(scheme.Scheme) + + var err error + if cfg, err = t.Start(); err != nil { + stdlog.Fatal(err) + } + + code := m.Run() + t.Stop() + os.Exit(code) +} + +// SetupTestReconcile returns a reconcile.Reconcile implementation that delegates to inner and +// writes the request to requests after Reconcile is finished. +func SetupTestReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, chan reconcile.Request) { + requests := make(chan reconcile.Request) + fn := reconcile.Func(func(req reconcile.Request) (reconcile.Result, error) { + result, err := inner.Reconcile(req) + requests <- req + return result, err + }) + return fn, requests +} + +// StartTestManager adds recFn +func StartTestManager(mgr manager.Manager, g *gomega.GomegaWithT) (chan struct{}, *sync.WaitGroup) { + stop := make(chan struct{}) + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + g.Expect(mgr.Start(stop)).NotTo(gomega.HaveOccurred()) + }() + return stop, wg +} diff --git a/pkg/controller/v1alpha3/experiment/experiment_controller_test.go b/pkg/controller/v1alpha3/experiment/experiment_controller_test.go new file mode 100644 index 00000000000..9b49ca9ef9c --- /dev/null +++ b/pkg/controller/v1alpha3/experiment/experiment_controller_test.go @@ -0,0 +1,89 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package experiment + +import ( + "testing" + "time" + + "github.com/onsi/gomega" + "golang.org/x/net/context" + appsv1 "k8s.io/api/apps/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + experimentsv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/experiment/v1alpha2" +) + +var c client.Client + +var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: "foo", Namespace: "default"}} +var depKey = types.NamespacedName{Name: "foo-deployment", Namespace: "default"} + +const timeout = time.Second * 5 + +func TestReconcile(t *testing.T) { + g := gomega.NewGomegaWithT(t) + instance := &experimentsv1alpha2.Experiment{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}} + + // Setup the Manager and Controller. Wrap the Controller Reconcile function so it writes each request to a + // channel when it is finished. + mgr, err := manager.New(cfg, manager.Options{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + c = mgr.GetClient() + + recFn, requests := SetupTestReconcile(newReconciler(mgr)) + g.Expect(add(mgr, recFn)).NotTo(gomega.HaveOccurred()) + + stopMgr, mgrStopped := StartTestManager(mgr, g) + + defer func() { + close(stopMgr) + mgrStopped.Wait() + }() + + // Create the Experiment object and expect the Reconcile and Deployment to be created + err = c.Create(context.TODO(), instance) + // The instance object may not be a valid object because it might be missing some required fields. + // Please modify the instance object by adding required fields and then remove the following if statement. + if apierrors.IsInvalid(err) { + t.Logf("failed to create object, got an invalid object error: %v", err) + return + } + g.Expect(err).NotTo(gomega.HaveOccurred()) + defer c.Delete(context.TODO(), instance) + g.Eventually(requests, timeout).Should(gomega.Receive(gomega.Equal(expectedRequest))) + + deploy := &appsv1.Deployment{} + g.Eventually(func() error { return c.Get(context.TODO(), depKey, deploy) }, timeout). + Should(gomega.Succeed()) + + // Delete the Deployment and expect Reconcile to be called for Deployment deletion + g.Expect(c.Delete(context.TODO(), deploy)).NotTo(gomega.HaveOccurred()) + g.Eventually(requests, timeout).Should(gomega.Receive(gomega.Equal(expectedRequest))) + g.Eventually(func() error { return c.Get(context.TODO(), depKey, deploy) }, timeout). + Should(gomega.Succeed()) + + // Manually delete Deployment since GC isn't enabled in the test control plane + g.Eventually(func() error { return c.Delete(context.TODO(), deploy) }, timeout). + Should(gomega.MatchError("deployments.apps \"foo-deployment\" not found")) + +} diff --git a/pkg/controller/v1alpha3/suggestion/suggestion_controller.go b/pkg/controller/v1alpha3/suggestion/suggestion_controller.go new file mode 100644 index 00000000000..813de264ded --- /dev/null +++ b/pkg/controller/v1alpha3/suggestion/suggestion_controller.go @@ -0,0 +1,168 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package suggestion + +import ( + "context" + "reflect" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/source" + + suggestionsv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/suggestion/v1alpha2" +) + +var log = logf.Log.WithName("controller") + +/** +* USER ACTION REQUIRED: This is a scaffold file intended for the user to modify with their own Controller +* business logic. Delete these comments after modifying this file.* + */ + +// Add creates a new Suggestion Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(mgr manager.Manager) error { + return add(mgr, newReconciler(mgr)) +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager) reconcile.Reconciler { + return &ReconcileSuggestion{Client: mgr.GetClient(), scheme: mgr.GetScheme()} +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New("suggestion-controller", mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + + // Watch for changes to Suggestion + err = c.Watch(&source.Kind{Type: &suggestionsv1alpha2.Suggestion{}}, &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } + + // TODO(user): Modify this to be the types you create + // Uncomment watch a Deployment created by Suggestion - change this for objects you create + err = c.Watch(&source.Kind{Type: &appsv1.Deployment{}}, &handler.EnqueueRequestForOwner{ + IsController: true, + OwnerType: &suggestionsv1alpha2.Suggestion{}, + }) + if err != nil { + return err + } + + return nil +} + +var _ reconcile.Reconciler = &ReconcileSuggestion{} + +// ReconcileSuggestion reconciles a Suggestion object +type ReconcileSuggestion struct { + client.Client + scheme *runtime.Scheme +} + +// Reconcile reads that state of the cluster for a Suggestion object and makes changes based on the state read +// and what is in the Suggestion.Spec +// TODO(user): Modify this Reconcile function to implement your Controller logic. The scaffolding writes +// a Deployment as an example +// Automatically generate RBAC rules to allow the Controller to read and write Deployments +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=deployments/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=suggestions.kubeflow.org,resources=suggestions,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=suggestions.kubeflow.org,resources=suggestions/status,verbs=get;update;patch +func (r *ReconcileSuggestion) Reconcile(request reconcile.Request) (reconcile.Result, error) { + // Fetch the Suggestion instance + instance := &suggestionsv1alpha2.Suggestion{} + err := r.Get(context.TODO(), request.NamespacedName, instance) + if err != nil { + if errors.IsNotFound(err) { + // Object not found, return. Created objects are automatically garbage collected. + // For additional cleanup logic use finalizers. + return reconcile.Result{}, nil + } + // Error reading the object - requeue the request. + return reconcile.Result{}, err + } + + // TODO(user): Change this to be the object type created by your controller + // Define the desired Deployment object + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name + "-deployment", + Namespace: instance.Namespace, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"deployment": instance.Name + "-deployment"}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"deployment": instance.Name + "-deployment"}}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, + } + if err := controllerutil.SetControllerReference(instance, deploy, r.scheme); err != nil { + return reconcile.Result{}, err + } + + // TODO(user): Change this for the object type created by your controller + // Check if the Deployment already exists + found := &appsv1.Deployment{} + err = r.Get(context.TODO(), types.NamespacedName{Name: deploy.Name, Namespace: deploy.Namespace}, found) + if err != nil && errors.IsNotFound(err) { + log.Info("Creating Deployment", "namespace", deploy.Namespace, "name", deploy.Name) + err = r.Create(context.TODO(), deploy) + return reconcile.Result{}, err + } else if err != nil { + return reconcile.Result{}, err + } + + // TODO(user): Change this for the object type created by your controller + // Update the found object and write the result back if there are any changes + if !reflect.DeepEqual(deploy.Spec, found.Spec) { + found.Spec = deploy.Spec + log.Info("Updating Deployment", "namespace", deploy.Namespace, "name", deploy.Name) + err = r.Update(context.TODO(), found) + if err != nil { + return reconcile.Result{}, err + } + } + return reconcile.Result{}, nil +} diff --git a/pkg/controller/v1alpha3/suggestion/suggestion_controller_suite_test.go b/pkg/controller/v1alpha3/suggestion/suggestion_controller_suite_test.go new file mode 100644 index 00000000000..ee4b9e30ab9 --- /dev/null +++ b/pkg/controller/v1alpha3/suggestion/suggestion_controller_suite_test.go @@ -0,0 +1,75 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package suggestion + +import ( + stdlog "log" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/kubeflow/katib/pkg/apis" + "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +var cfg *rest.Config + +func TestMain(m *testing.M) { + t := &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, + } + apis.AddToScheme(scheme.Scheme) + + var err error + if cfg, err = t.Start(); err != nil { + stdlog.Fatal(err) + } + + code := m.Run() + t.Stop() + os.Exit(code) +} + +// SetupTestReconcile returns a reconcile.Reconcile implementation that delegates to inner and +// writes the request to requests after Reconcile is finished. +func SetupTestReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, chan reconcile.Request) { + requests := make(chan reconcile.Request) + fn := reconcile.Func(func(req reconcile.Request) (reconcile.Result, error) { + result, err := inner.Reconcile(req) + requests <- req + return result, err + }) + return fn, requests +} + +// StartTestManager adds recFn +func StartTestManager(mgr manager.Manager, g *gomega.GomegaWithT) (chan struct{}, *sync.WaitGroup) { + stop := make(chan struct{}) + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + g.Expect(mgr.Start(stop)).NotTo(gomega.HaveOccurred()) + }() + return stop, wg +} diff --git a/pkg/controller/v1alpha3/suggestion/suggestion_controller_test.go b/pkg/controller/v1alpha3/suggestion/suggestion_controller_test.go new file mode 100644 index 00000000000..dd9c4fe31ee --- /dev/null +++ b/pkg/controller/v1alpha3/suggestion/suggestion_controller_test.go @@ -0,0 +1,89 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package suggestion + +import ( + "testing" + "time" + + "github.com/onsi/gomega" + "golang.org/x/net/context" + appsv1 "k8s.io/api/apps/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + suggestionsv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/suggestions/v1alpha2" +) + +var c client.Client + +var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: "foo", Namespace: "default"}} +var depKey = types.NamespacedName{Name: "foo-deployment", Namespace: "default"} + +const timeout = time.Second * 5 + +func TestReconcile(t *testing.T) { + g := gomega.NewGomegaWithT(t) + instance := &suggestionsv1alpha2.Suggestion{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}} + + // Setup the Manager and Controller. Wrap the Controller Reconcile function so it writes each request to a + // channel when it is finished. + mgr, err := manager.New(cfg, manager.Options{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + c = mgr.GetClient() + + recFn, requests := SetupTestReconcile(newReconciler(mgr)) + g.Expect(add(mgr, recFn)).NotTo(gomega.HaveOccurred()) + + stopMgr, mgrStopped := StartTestManager(mgr, g) + + defer func() { + close(stopMgr) + mgrStopped.Wait() + }() + + // Create the Suggestion object and expect the Reconcile and Deployment to be created + err = c.Create(context.TODO(), instance) + // The instance object may not be a valid object because it might be missing some required fields. + // Please modify the instance object by adding required fields and then remove the following if statement. + if apierrors.IsInvalid(err) { + t.Logf("failed to create object, got an invalid object error: %v", err) + return + } + g.Expect(err).NotTo(gomega.HaveOccurred()) + defer c.Delete(context.TODO(), instance) + g.Eventually(requests, timeout).Should(gomega.Receive(gomega.Equal(expectedRequest))) + + deploy := &appsv1.Deployment{} + g.Eventually(func() error { return c.Get(context.TODO(), depKey, deploy) }, timeout). + Should(gomega.Succeed()) + + // Delete the Deployment and expect Reconcile to be called for Deployment deletion + g.Expect(c.Delete(context.TODO(), deploy)).NotTo(gomega.HaveOccurred()) + g.Eventually(requests, timeout).Should(gomega.Receive(gomega.Equal(expectedRequest))) + g.Eventually(func() error { return c.Get(context.TODO(), depKey, deploy) }, timeout). + Should(gomega.Succeed()) + + // Manually delete Deployment since GC isn't enabled in the test control plane + g.Eventually(func() error { return c.Delete(context.TODO(), deploy) }, timeout). + Should(gomega.MatchError("deployments.apps \"foo-deployment\" not found")) + +} diff --git a/pkg/controller/v1alpha3/trial/trial_controller.go b/pkg/controller/v1alpha3/trial/trial_controller.go new file mode 100644 index 00000000000..d017c8090e5 --- /dev/null +++ b/pkg/controller/v1alpha3/trial/trial_controller.go @@ -0,0 +1,168 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package trial + +import ( + "context" + "reflect" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/source" + + trialv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/trial/v1alpha2" +) + +var log = logf.Log.WithName("controller") + +/** +* USER ACTION REQUIRED: This is a scaffold file intended for the user to modify with their own Controller +* business logic. Delete these comments after modifying this file.* + */ + +// Add creates a new Trial Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(mgr manager.Manager) error { + return add(mgr, newReconciler(mgr)) +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager) reconcile.Reconciler { + return &ReconcileTrial{Client: mgr.GetClient(), scheme: mgr.GetScheme()} +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New("trial-controller", mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + + // Watch for changes to Trial + err = c.Watch(&source.Kind{Type: &trialv1alpha2.Trial{}}, &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } + + // TODO(user): Modify this to be the types you create + // Uncomment watch a Deployment created by Trial - change this for objects you create + err = c.Watch(&source.Kind{Type: &appsv1.Deployment{}}, &handler.EnqueueRequestForOwner{ + IsController: true, + OwnerType: &trialv1alpha2.Trial{}, + }) + if err != nil { + return err + } + + return nil +} + +var _ reconcile.Reconciler = &ReconcileTrial{} + +// ReconcileTrial reconciles a Trial object +type ReconcileTrial struct { + client.Client + scheme *runtime.Scheme +} + +// Reconcile reads that state of the cluster for a Trial object and makes changes based on the state read +// and what is in the Trial.Spec +// TODO(user): Modify this Reconcile function to implement your Controller logic. The scaffolding writes +// a Deployment as an example +// Automatically generate RBAC rules to allow the Controller to read and write Deployments +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=deployments/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=trials.kubeflow.org,resources=trials,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=trials.kubeflow.org,resources=trials/status,verbs=get;update;patch +func (r *ReconcileTrial) Reconcile(request reconcile.Request) (reconcile.Result, error) { + // Fetch the Trial instance + instance := &trialv1alpha2.Trial{} + err := r.Get(context.TODO(), request.NamespacedName, instance) + if err != nil { + if errors.IsNotFound(err) { + // Object not found, return. Created objects are automatically garbage collected. + // For additional cleanup logic use finalizers. + return reconcile.Result{}, nil + } + // Error reading the object - requeue the request. + return reconcile.Result{}, err + } + + // TODO(user): Change this to be the object type created by your controller + // Define the desired Deployment object + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name + "-deployment", + Namespace: instance.Namespace, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"deployment": instance.Name + "-deployment"}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"deployment": instance.Name + "-deployment"}}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, + } + if err := controllerutil.SetControllerReference(instance, deploy, r.scheme); err != nil { + return reconcile.Result{}, err + } + + // TODO(user): Change this for the object type created by your controller + // Check if the Deployment already exists + found := &appsv1.Deployment{} + err = r.Get(context.TODO(), types.NamespacedName{Name: deploy.Name, Namespace: deploy.Namespace}, found) + if err != nil && errors.IsNotFound(err) { + log.Info("Creating Deployment", "namespace", deploy.Namespace, "name", deploy.Name) + err = r.Create(context.TODO(), deploy) + return reconcile.Result{}, err + } else if err != nil { + return reconcile.Result{}, err + } + + // TODO(user): Change this for the object type created by your controller + // Update the found object and write the result back if there are any changes + if !reflect.DeepEqual(deploy.Spec, found.Spec) { + found.Spec = deploy.Spec + log.Info("Updating Deployment", "namespace", deploy.Namespace, "name", deploy.Name) + err = r.Update(context.TODO(), found) + if err != nil { + return reconcile.Result{}, err + } + } + return reconcile.Result{}, nil +} diff --git a/pkg/controller/v1alpha3/trial/trial_controller_suite_test.go b/pkg/controller/v1alpha3/trial/trial_controller_suite_test.go new file mode 100644 index 00000000000..dbc8fd9ce0d --- /dev/null +++ b/pkg/controller/v1alpha3/trial/trial_controller_suite_test.go @@ -0,0 +1,75 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package trial + +import ( + stdlog "log" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/kubeflow/katib/pkg/apis" + "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +var cfg *rest.Config + +func TestMain(m *testing.M) { + t := &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, + } + apis.AddToScheme(scheme.Scheme) + + var err error + if cfg, err = t.Start(); err != nil { + stdlog.Fatal(err) + } + + code := m.Run() + t.Stop() + os.Exit(code) +} + +// SetupTestReconcile returns a reconcile.Reconcile implementation that delegates to inner and +// writes the request to requests after Reconcile is finished. +func SetupTestReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, chan reconcile.Request) { + requests := make(chan reconcile.Request) + fn := reconcile.Func(func(req reconcile.Request) (reconcile.Result, error) { + result, err := inner.Reconcile(req) + requests <- req + return result, err + }) + return fn, requests +} + +// StartTestManager adds recFn +func StartTestManager(mgr manager.Manager, g *gomega.GomegaWithT) (chan struct{}, *sync.WaitGroup) { + stop := make(chan struct{}) + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + g.Expect(mgr.Start(stop)).NotTo(gomega.HaveOccurred()) + }() + return stop, wg +} diff --git a/pkg/controller/v1alpha3/trial/trial_controller_test.go b/pkg/controller/v1alpha3/trial/trial_controller_test.go new file mode 100644 index 00000000000..e753339e4ae --- /dev/null +++ b/pkg/controller/v1alpha3/trial/trial_controller_test.go @@ -0,0 +1,89 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package trial + +import ( + "testing" + "time" + + "github.com/onsi/gomega" + "golang.org/x/net/context" + appsv1 "k8s.io/api/apps/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + trialv1alpha2 "github.com/kubeflow/katib/pkg/api/operators/apis/trial/v1alpha2" +) + +var c client.Client + +var expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: "foo", Namespace: "default"}} +var depKey = types.NamespacedName{Name: "foo-deployment", Namespace: "default"} + +const timeout = time.Second * 5 + +func TestReconcile(t *testing.T) { + g := gomega.NewGomegaWithT(t) + instance := &trialv1alpha2.Trial{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}} + + // Setup the Manager and Controller. Wrap the Controller Reconcile function so it writes each request to a + // channel when it is finished. + mgr, err := manager.New(cfg, manager.Options{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + c = mgr.GetClient() + + recFn, requests := SetupTestReconcile(newReconciler(mgr)) + g.Expect(add(mgr, recFn)).NotTo(gomega.HaveOccurred()) + + stopMgr, mgrStopped := StartTestManager(mgr, g) + + defer func() { + close(stopMgr) + mgrStopped.Wait() + }() + + // Create the Trial object and expect the Reconcile and Deployment to be created + err = c.Create(context.TODO(), instance) + // The instance object may not be a valid object because it might be missing some required fields. + // Please modify the instance object by adding required fields and then remove the following if statement. + if apierrors.IsInvalid(err) { + t.Logf("failed to create object, got an invalid object error: %v", err) + return + } + g.Expect(err).NotTo(gomega.HaveOccurred()) + defer c.Delete(context.TODO(), instance) + g.Eventually(requests, timeout).Should(gomega.Receive(gomega.Equal(expectedRequest))) + + deploy := &appsv1.Deployment{} + g.Eventually(func() error { return c.Get(context.TODO(), depKey, deploy) }, timeout). + Should(gomega.Succeed()) + + // Delete the Deployment and expect Reconcile to be called for Deployment deletion + g.Expect(c.Delete(context.TODO(), deploy)).NotTo(gomega.HaveOccurred()) + g.Eventually(requests, timeout).Should(gomega.Receive(gomega.Equal(expectedRequest))) + g.Eventually(func() error { return c.Get(context.TODO(), depKey, deploy) }, timeout). + Should(gomega.Succeed()) + + // Manually delete Deployment since GC isn't enabled in the test control plane + g.Eventually(func() error { return c.Delete(context.TODO(), deploy) }, timeout). + Should(gomega.MatchError("deployments.apps \"foo-deployment\" not found")) + +} diff --git a/prow_config.yaml b/prow_config.yaml index 50cc6540342..3a6bd1a7e26 100644 --- a/prow_config.yaml +++ b/prow_config.yaml @@ -6,6 +6,10 @@ workflows: name: e2e job_types: - presubmit + include_dirs: + - pkg/* + - cmd/* + - test/* params: registry: "gcr.io/kubeflow-ci" # The postsubmit run publishes the docker images to gcr.io/kubeflow-images-public @@ -14,5 +18,9 @@ workflows: name: e2e-release job_types: - postsubmit + include_dirs: + - pkg/* + - cmd/* + - test/* params: - registry: "gcr.io/kubeflow-images-public" \ No newline at end of file + registry: "gcr.io/kubeflow-images-public"