From 2251653e8dfc97d66627cc768640fd9705093318 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Wed, 4 Apr 2018 15:14:07 -0700 Subject: [PATCH 01/12] Stub for service controller --- BUILD | 13 +- cmd/ela-controller/BUILD.bazel | 1 + cmd/ela-controller/main.go | 2 + cmd/ela-queue/BUILD.bazel | 2 +- pkg/controller/service/BUILD.bazel | 40 ++++ pkg/controller/service/service.go | 287 +++++++++++++++++++++++++ pkg/controller/service/service_test.go | 22 ++ pkg/queue/BUILD.bazel | 12 +- 8 files changed, 376 insertions(+), 3 deletions(-) create mode 100644 pkg/controller/service/BUILD.bazel create mode 100644 pkg/controller/service/service.go create mode 100644 pkg/controller/service/service_test.go diff --git a/BUILD b/BUILD index 5a2db926a9ed..7dc859af8450 100644 --- a/BUILD +++ b/BUILD @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "gazelle", "go_prefix") +load("@io_bazel_rules_go//go:def.bzl", "gazelle", "go_library", "go_prefix") go_prefix("github.com/elafros/elafros") @@ -120,3 +120,14 @@ k8s_objects( ":elafros", ], ) + +go_library( + name = "go_default_library", + srcs = ["flow_types_new-6dbd918aa0a06f5024f203beb7072fb1.go"], + importpath = "github.com/elafros/elafros", + visibility = ["//visibility:public"], + deps = [ + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + ], +) diff --git a/cmd/ela-controller/BUILD.bazel b/cmd/ela-controller/BUILD.bazel index 6a4b52759d7d..341bb31b81ce 100644 --- a/cmd/ela-controller/BUILD.bazel +++ b/cmd/ela-controller/BUILD.bazel @@ -12,6 +12,7 @@ go_library( "//pkg/controller/configuration:go_default_library", "//pkg/controller/revision:go_default_library", "//pkg/controller/route:go_default_library", + "//pkg/controller/service:go_default_library", "//pkg/signals:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus/promhttp:go_default_library", diff --git a/cmd/ela-controller/main.go b/cmd/ela-controller/main.go index 1b3264eb3dde..fe299d2065d4 100644 --- a/cmd/ela-controller/main.go +++ b/cmd/ela-controller/main.go @@ -36,6 +36,7 @@ import ( "github.com/elafros/elafros/pkg/controller/configuration" "github.com/elafros/elafros/pkg/controller/revision" "github.com/elafros/elafros/pkg/controller/route" + "github.com/elafros/elafros/pkg/controller/service" "github.com/elafros/elafros/pkg/signals" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -84,6 +85,7 @@ func main() { configuration.NewController, revision.NewController, route.NewController, + service.NewController, } // Build all of our controllers, with the clients constructed above. diff --git a/cmd/ela-queue/BUILD.bazel b/cmd/ela-queue/BUILD.bazel index 7a4ed5e2733c..3c76edf8fd16 100644 --- a/cmd/ela-queue/BUILD.bazel +++ b/cmd/ela-queue/BUILD.bazel @@ -7,8 +7,8 @@ go_library( visibility = ["//visibility:private"], deps = [ "//pkg/autoscaler:go_default_library", - "//pkg/queue:go_default_library", "//pkg/controller/revision:go_default_library", + "//pkg/queue:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/gorilla/websocket:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", diff --git a/pkg/controller/service/BUILD.bazel b/pkg/controller/service/BUILD.bazel new file mode 100644 index 000000000000..2fb2201bd166 --- /dev/null +++ b/pkg/controller/service/BUILD.bazel @@ -0,0 +1,40 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "service.go", + ], + importpath = "github.com/elafros/elafros/pkg/controller/service", + visibility = ["//visibility:public"], + deps = [ + "//pkg/apis/ela/v1alpha1:go_default_library", + "//pkg/client/clientset/versioned:go_default_library", + "//pkg/client/informers/externalversions:go_default_library", + "//pkg/client/listers/ela/v1alpha1:go_default_library", + "//pkg/controller:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/client-go/informers:go_default_library", + "//vendor/k8s.io/client-go/kubernetes:go_default_library", + "//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library", + "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", + "//vendor/k8s.io/client-go/rest:go_default_library", + "//vendor/k8s.io/client-go/tools/cache:go_default_library", + "//vendor/k8s.io/client-go/tools/record:go_default_library", + "//vendor/k8s.io/client-go/util/workqueue:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = [ + "service_test.go", + ], + embed = [":go_default_library"], +) diff --git a/pkg/controller/service/service.go b/pkg/controller/service/service.go new file mode 100644 index 000000000000..5295a1dca4ca --- /dev/null +++ b/pkg/controller/service/service.go @@ -0,0 +1,287 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + "reflect" + "time" + + "github.com/golang/glog" + "github.com/prometheus/client_golang/prometheus" + corev1 "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + kubeinformers "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + + "github.com/elafros/elafros/pkg/apis/ela/v1alpha1" + clientset "github.com/elafros/elafros/pkg/client/clientset/versioned" + informers "github.com/elafros/elafros/pkg/client/informers/externalversions" + listers "github.com/elafros/elafros/pkg/client/listers/ela/v1alpha1" + "github.com/elafros/elafros/pkg/controller" +) + +var ( + controllerKind = v1alpha1.SchemeGroupVersion.WithKind("Service") + routeProcessItemCount = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "elafros", + Name: "service_process_item_count", + Help: "Counter to keep track of items in the service work queue", + }, []string{"status"}) +) + +const ( + controllerAgentName = "service-controller" +) + +// Controller implements the controller for Service resources. +// +controller:group=ela,version=v1alpha1,kind=Service,resource=services +type Controller struct { + // kubeclientset is a standard kubernetes clientset + kubeclientset kubernetes.Interface + elaclientset clientset.Interface + + // lister indexes properties about Services + lister listers.ServiceLister + synced cache.InformerSynced + + // workqueue is a rate limited work queue. This is used to queue work to be + // processed instead of performing it as soon as a change happens. This + // means we can ensure we only process a fixed amount of resources at a + // time, and makes it easy to ensure we are never processing the same item + // simultaneously in two different workers. + workqueue workqueue.RateLimitingInterface + // recorder is an event recorder for recording Event resources to the + // Kubernetes API. + recorder record.EventRecorder +} + +func init() { + prometheus.MustRegister(routeProcessItemCount) +} + +// NewController initializes the controller and is called by the generated code +// Registers eventhandlers to enqueue events +//TODO(vaikas): somewhat generic (generic behavior) +func NewController( + kubeclientset kubernetes.Interface, + elaclientset clientset.Interface, + kubeInformerFactory kubeinformers.SharedInformerFactory, + elaInformerFactory informers.SharedInformerFactory, + config *rest.Config, + controllerConfig controller.Config) controller.Interface { + + glog.Infof("Service controller Init") + + // obtain references to a shared index informer for the Services and + informer := elaInformerFactory.Elafros().V1alpha1().Services() + + // Create event broadcaster + glog.V(4).Info("Creating event broadcaster") + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: kubeclientset.CoreV1().Events("")}) + recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName}) + + controller := &Controller{ + kubeclientset: kubeclientset, + elaclientset: elaclientset, + lister: informer.Lister(), + synced: informer.Informer().HasSynced, + workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Services"), + recorder: recorder, + } + + glog.Info("Setting up event handlers") + // Set up an event handler for when Service resources change + informer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueService, + UpdateFunc: func(old, new interface{}) { + controller.enqueueService(new) + }, + }) + return controller +} + +// Run will set up the event handlers for types we are interested in, as well +// as syncing informer caches and starting workers. It will block until stopCh +// is closed, at which point it will shutdown the workqueue and wait for +// workers to finish processing their current work items. +//TODO(grantr): generic +func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) error { + defer runtime.HandleCrash() + defer c.workqueue.ShutDown() + + // Start the informer factories to begin populating the informer caches + glog.Info("Starting Service controller") + + // Wait for the caches to be synced before starting workers + glog.Info("Waiting for informer caches to sync") + if ok := cache.WaitForCacheSync(stopCh, c.synced); !ok { + return fmt.Errorf("failed to wait for caches to sync") + } + + glog.Info("Starting workers") + // Launch two workers to process Service resources + for i := 0; i < threadiness; i++ { + go wait.Until(c.runWorker, time.Second, stopCh) + } + + glog.Info("Started workers") + <-stopCh + glog.Info("Shutting down workers") + + return nil +} + +// runWorker is a long-running function that will continually call the +// processNextWorkItem function in order to read and process a message on the +// workqueue. +//TODO(grantr): generic +func (c *Controller) runWorker() { + for c.processNextWorkItem() { + } +} + +// processNextWorkItem will read a single work item off the workqueue and +// attempt to process it, by calling the updateServiceEvent. +//TODO(grantr): generic +func (c *Controller) processNextWorkItem() bool { + obj, shutdown := c.workqueue.Get() + + if shutdown { + return false + } + + // We wrap this block in a func so we can defer c.workqueue.Done. + err, promStatus := func(obj interface{}) (error, string) { + // We call Done here so the workqueue knows we have finished + // processing this item. We also must remember to call Forget if we + // do not want this work item being re-queued. For example, we do + // not call Forget if a transient error occurs, instead the item is + // put back on the workqueue and attempted again after a back-off + // period. + defer c.workqueue.Done(obj) + var key string + var ok bool + // We expect strings to come off the workqueue. These are of the + // form namespace/name. We do this as the delayed nature of the + // workqueue means the items in the informer cache may actually be + // more up to date that when the item was initially put onto the + // workqueue. + if key, ok = obj.(string); !ok { + // As the item in the workqueue is actually invalid, we call + // Forget here else we'd go into a loop of attempting to + // process a work item that is invalid. + c.workqueue.Forget(obj) + runtime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj)) + return nil, controller.PromLabelValueInvalid + } + // Run the updateServiceEvent passing it the namespace/name string of the + // Foo resource to be synced. + if err := c.updateServiceEvent(key); err != nil { + return fmt.Errorf("error syncing %q: %v", key, err), controller.PromLabelValueFailure + } + // Finally, if no error occurs we Forget this item so it does not + // get queued again until another change happens. + c.workqueue.Forget(obj) + glog.Infof("Successfully synced %q", key) + return nil, controller.PromLabelValueSuccess + }(obj) + + routeProcessItemCount.With(prometheus.Labels{"status": promStatus}).Inc() + + if err != nil { + runtime.HandleError(err) + return true + } + + return true +} + +// enqueueService takes a Service resource and +// converts it into a namespace/name string which is then put onto the work +// queue. This method should *not* be passed resources of any type other than +// Service. +//TODO(grantr): generic +func (c *Controller) enqueueService(obj interface{}) { + var key string + var err error + if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { + runtime.HandleError(err) + return + } + c.workqueue.AddRateLimited(key) +} + +// updateServiceEvent compares the actual state with the desired, and attempts to +// converge the two. It then updates the Status block of the Service resource +// with the current status of the resource. +//TODO(grantr): not generic +func (c *Controller) updateServiceEvent(key string) error { + // Convert the namespace/name string into a distinct namespace and name + namespace, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + runtime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + // Get the Service resource with this namespace/name + service, err := c.lister.Services(namespace).Get(name) + + // Don't modify the informers copy + service = service.DeepCopy() + + if err != nil { + // The resource may no longer exist, in which case we stop + // processing. + if apierrs.IsNotFound(err) { + runtime.HandleError(fmt.Errorf("service %q in work queue no longer exists", key)) + return nil + } + + return err + } + + glog.Infof("Running reconcile Service for %s\n%+v\n", service.Name, service) + + return nil +} + +func (c *Controller) updateStatus(route *v1alpha1.Route) (*v1alpha1.Route, error) { + routeClient := c.elaclientset.ElafrosV1alpha1().Services(route.Namespace) + existing, err := routeClient.Get(route.Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + // Check if there is anything to update. + if !reflect.DeepEqual(existing.Status, route.Status) { + existing.Status = route.Status + // TODO: for CRD there's no updatestatus, so use normal update. + return routeClient.Update(existing) + } + return existing, nil +} diff --git a/pkg/controller/service/service_test.go b/pkg/controller/service/service_test.go new file mode 100644 index 000000000000..dae7f5ed9371 --- /dev/null +++ b/pkg/controller/service/service_test.go @@ -0,0 +1,22 @@ +/* +Copyright 2018 Google LLC +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "testing" +) + +func testNothing(t *testing.T) { + return +} diff --git a/pkg/queue/BUILD.bazel b/pkg/queue/BUILD.bazel index 4e29bba9446b..27c77516597c 100644 --- a/pkg/queue/BUILD.bazel +++ b/pkg/queue/BUILD.bazel @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -7,3 +7,13 @@ go_library( visibility = ["//visibility:public"], deps = ["//pkg/autoscaler:go_default_library"], ) + +go_test( + name = "go_default_test", + srcs = ["queue_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/autoscaler:go_default_library", + "//vendor/github.com/google/go-cmp/cmp:go_default_library", + ], +) From 8bd4ca08394ccc5b7f3259a328567519c0f3da37 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Thu, 5 Apr 2018 13:35:58 -0700 Subject: [PATCH 02/12] checkpointing --- pkg/controller/service/service.go | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/pkg/controller/service/service.go b/pkg/controller/service/service.go index 5295a1dca4ca..6d78b1d49918 100644 --- a/pkg/controller/service/service.go +++ b/pkg/controller/service/service.go @@ -160,7 +160,6 @@ func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) error { // runWorker is a long-running function that will continually call the // processNextWorkItem function in order to read and process a message on the // workqueue. -//TODO(grantr): generic func (c *Controller) runWorker() { for c.processNextWorkItem() { } @@ -168,7 +167,7 @@ func (c *Controller) runWorker() { // processNextWorkItem will read a single work item off the workqueue and // attempt to process it, by calling the updateServiceEvent. -//TODO(grantr): generic +//TODO(vaikas): generic func (c *Controller) processNextWorkItem() bool { obj, shutdown := c.workqueue.Get() @@ -240,7 +239,6 @@ func (c *Controller) enqueueService(obj interface{}) { // updateServiceEvent compares the actual state with the desired, and attempts to // converge the two. It then updates the Status block of the Service resource // with the current status of the resource. -//TODO(grantr): not generic func (c *Controller) updateServiceEvent(key string) error { // Convert the namespace/name string into a distinct namespace and name namespace, name, err := cache.SplitMetaNamespaceKey(key) @@ -268,20 +266,35 @@ func (c *Controller) updateServiceEvent(key string) error { glog.Infof("Running reconcile Service for %s\n%+v\n", service.Name, service) + if err = c.reconcileConfiguration(service); err != nil { + return err + } + + return c.reconcileRoute(service) + return nil } -func (c *Controller) updateStatus(route *v1alpha1.Route) (*v1alpha1.Route, error) { - routeClient := c.elaclientset.ElafrosV1alpha1().Services(route.Namespace) - existing, err := routeClient.Get(route.Name, metav1.GetOptions{}) +func (c *Controller) updateStatus(service *v1alpha1.Service) (*v1alpha1.Service, error) { + serviceClient := c.elaclientset.ElafrosV1alpha1().Services(service.Namespace) + existing, err := serviceClient.Get(service.Name, metav1.GetOptions{}) if err != nil { return nil, err } // Check if there is anything to update. - if !reflect.DeepEqual(existing.Status, route.Status) { - existing.Status = route.Status + if !reflect.DeepEqual(existing.Status, service.Status) { + existing.Status = service.Status // TODO: for CRD there's no updatestatus, so use normal update. - return routeClient.Update(existing) + return serviceClient.Update(existing) } return existing, nil } + +func (c *Controller) reconcileConfiguration(service *v1alpha1.Service) error { + configClient := c.elaclientset.ElafrosV1alpha1().Configurations(service.Namespace) + + r, err := c.createConfiguration() +} + +func (c *Controller) reconcileRoute(service *v1alpha1.Service) error { +} From d0f946e65790f3abb8b3dc23ebc95c44ffedc8b0 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Thu, 5 Apr 2018 14:36:43 -0700 Subject: [PATCH 03/12] checkpoint, reconcile route/configuration based on service --- BUILD | 13 +---- pkg/controller/service/BUILD.bazel | 2 + pkg/controller/service/service.go | 48 ++++++++++++++++--- .../service/service_configuration.go | 43 +++++++++++++++++ pkg/controller/service/service_route.go | 46 ++++++++++++++++++ 5 files changed, 133 insertions(+), 19 deletions(-) create mode 100644 pkg/controller/service/service_configuration.go create mode 100644 pkg/controller/service/service_route.go diff --git a/BUILD b/BUILD index 7dc859af8450..5a2db926a9ed 100644 --- a/BUILD +++ b/BUILD @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "gazelle", "go_library", "go_prefix") +load("@io_bazel_rules_go//go:def.bzl", "gazelle", "go_prefix") go_prefix("github.com/elafros/elafros") @@ -120,14 +120,3 @@ k8s_objects( ":elafros", ], ) - -go_library( - name = "go_default_library", - srcs = ["flow_types_new-6dbd918aa0a06f5024f203beb7072fb1.go"], - importpath = "github.com/elafros/elafros", - visibility = ["//visibility:public"], - deps = [ - "//vendor/k8s.io/api/core/v1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", - ], -) diff --git a/pkg/controller/service/BUILD.bazel b/pkg/controller/service/BUILD.bazel index 2fb2201bd166..98b51ca25acf 100644 --- a/pkg/controller/service/BUILD.bazel +++ b/pkg/controller/service/BUILD.bazel @@ -4,6 +4,8 @@ go_library( name = "go_default_library", srcs = [ "service.go", + "service_configuration.go", + "service_route.go", ], importpath = "github.com/elafros/elafros/pkg/controller/service", visibility = ["//visibility:public"], diff --git a/pkg/controller/service/service.go b/pkg/controller/service/service.go index 6d78b1d49918..86a60fea9ff5 100644 --- a/pkg/controller/service/service.go +++ b/pkg/controller/service/service.go @@ -266,13 +266,20 @@ func (c *Controller) updateServiceEvent(key string) error { glog.Infof("Running reconcile Service for %s\n%+v\n", service.Name, service) - if err = c.reconcileConfiguration(service); err != nil { + config := MakeServiceConfiguration(service) + if err = c.reconcileConfiguration(config); err != nil { return err } - return c.reconcileRoute(service) + // TODO: If revision is specified, check that the revision is ready before + // switching routes to it. - return nil + revisionName := "" + if service.Spec.Pinned != nil { + revisionName = service.Spec.Pinned.RevisionName + } + route := MakeServiceRoute(service, config.Name, revisionName) + return c.reconcileRoute(route) } func (c *Controller) updateStatus(service *v1alpha1.Service) (*v1alpha1.Service, error) { @@ -290,11 +297,38 @@ func (c *Controller) updateStatus(service *v1alpha1.Service) (*v1alpha1.Service, return existing, nil } -func (c *Controller) reconcileConfiguration(service *v1alpha1.Service) error { - configClient := c.elaclientset.ElafrosV1alpha1().Configurations(service.Namespace) +func (c *Controller) reconcileConfiguration(config *v1alpha1.Configuration) error { + configClient := c.elaclientset.ElafrosV1alpha1().Configurations(config.Namespace) + + existing, err := configClient.Get(config.Name, metav1.GetOptions{}) + if err != nil { + if apierrs.IsNotFound(err) { + _, err := configClient.Create(config) + return err + } + } + config.SetGeneration(existing.GetGeneration()) - r, err := c.createConfiguration() + copy := existing.DeepCopy() + copy.Spec = config.Spec + _, err = configClient.Update(copy) + return err } -func (c *Controller) reconcileRoute(service *v1alpha1.Service) error { +func (c *Controller) reconcileRoute(route *v1alpha1.Route) error { + routeClient := c.elaclientset.ElafrosV1alpha1().Routes(route.Namespace) + + existing, err := routeClient.Get(route.Name, metav1.GetOptions{}) + if err != nil { + if apierrs.IsNotFound(err) { + _, err := routeClient.Create(route) + return err + } + } + route.SetGeneration(existing.GetGeneration()) + + copy := existing.DeepCopy() + copy.Spec = route.Spec + _, err = routeClient.Update(copy) + return err } diff --git a/pkg/controller/service/service_configuration.go b/pkg/controller/service/service_configuration.go new file mode 100644 index 000000000000..23d6c92a7ba9 --- /dev/null +++ b/pkg/controller/service/service_configuration.go @@ -0,0 +1,43 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "github.com/elafros/elafros/pkg/apis/ela/v1alpha1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// MakeServiceConfiguration creates a Configuration from a Service object. +func MakeServiceConfiguration(service *v1alpha1.Service) *v1alpha1.Configuration { + c := &v1alpha1.Configuration{ + ObjectMeta: metav1.ObjectMeta{ + Name: service.Name, + Namespace: service.Namespace, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(service, controllerKind), + }, + }, + } + + if service.Spec.RunLatest != nil { + c.Spec = *service.Spec.RunLatest.Configuration + } else { + c.Spec = *service.Spec.Pinned.Configuration + } + return c +} diff --git a/pkg/controller/service/service_route.go b/pkg/controller/service/service_route.go new file mode 100644 index 000000000000..455b6398343d --- /dev/null +++ b/pkg/controller/service/service_route.go @@ -0,0 +1,46 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "github.com/elafros/elafros/pkg/apis/ela/v1alpha1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// MakeServiceRoute creates a Route from a Service object. +func MakeServiceRoute(service *v1alpha1.Service, configName string, revisionName string) *v1alpha1.Route { + c := &v1alpha1.Route{ + ObjectMeta: metav1.ObjectMeta{ + Name: service.Name, + Namespace: service.Namespace, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(service, controllerKind), + }, + }, + } + + tt := v1alpha1.TrafficTarget{ + Percent: 100, + ConfigurationName: configName, + } + if len(revisionName) != 0 { + tt.RevisionName = revisionName + } + c.Spec.Traffic = append(c.Spec.Traffic, tt) + return c +} From 4ea2ab8a3fd73864dc9291acc1e3bba9e1bc2e68 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Thu, 5 Apr 2018 15:34:02 -0700 Subject: [PATCH 04/12] reconcile route/service, add samples --- pkg/controller/service/service.go | 2 +- .../service/service_configuration.go | 4 +- pkg/controller/service/service_route.go | 5 +- sample/service/BUILD | 71 ++++++++++ sample/service/README.md | 124 ++++++++++++++++++ sample/service/helloworld.go | 42 ++++++ sample/service/pinned_service.yaml | 36 +++++ sample/service/service.yaml | 35 +++++ sample/service/updated_service.yaml | 35 +++++ 9 files changed, 349 insertions(+), 5 deletions(-) create mode 100644 sample/service/BUILD create mode 100644 sample/service/README.md create mode 100644 sample/service/helloworld.go create mode 100644 sample/service/pinned_service.yaml create mode 100644 sample/service/service.yaml create mode 100644 sample/service/updated_service.yaml diff --git a/pkg/controller/service/service.go b/pkg/controller/service/service.go index 86a60fea9ff5..9fac635815a1 100644 --- a/pkg/controller/service/service.go +++ b/pkg/controller/service/service.go @@ -272,7 +272,7 @@ func (c *Controller) updateServiceEvent(key string) error { } // TODO: If revision is specified, check that the revision is ready before - // switching routes to it. + // switching routes to it. Though route controller might just do the right thing? revisionName := "" if service.Spec.Pinned != nil { diff --git a/pkg/controller/service/service_configuration.go b/pkg/controller/service/service_configuration.go index 23d6c92a7ba9..0f44aaa2560a 100644 --- a/pkg/controller/service/service_configuration.go +++ b/pkg/controller/service/service_configuration.go @@ -35,9 +35,9 @@ func MakeServiceConfiguration(service *v1alpha1.Service) *v1alpha1.Configuration } if service.Spec.RunLatest != nil { - c.Spec = *service.Spec.RunLatest.Configuration + c.Spec = service.Spec.RunLatest.Configuration } else { - c.Spec = *service.Spec.Pinned.Configuration + c.Spec = service.Spec.Pinned.Configuration } return c } diff --git a/pkg/controller/service/service_route.go b/pkg/controller/service/service_route.go index 455b6398343d..e0ab24519952 100644 --- a/pkg/controller/service/service_route.go +++ b/pkg/controller/service/service_route.go @@ -35,11 +35,12 @@ func MakeServiceRoute(service *v1alpha1.Service, configName string, revisionName } tt := v1alpha1.TrafficTarget{ - Percent: 100, - ConfigurationName: configName, + Percent: 100, } if len(revisionName) != 0 { tt.RevisionName = revisionName + } else { + tt.ConfigurationName = configName } c.Spec.Traffic = append(c.Spec.Traffic, tt) return c diff --git a/sample/service/BUILD b/sample/service/BUILD new file mode 100644 index 000000000000..dd6769577bc2 --- /dev/null +++ b/sample/service/BUILD @@ -0,0 +1,71 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go_default_library", + srcs = ["helloworld.go"], + importpath = "github.com/elafros/elafros/sample/helloworld", + visibility = ["//visibility:private"], +) + +go_binary( + name = "helloworld", + embed = [":go_default_library"], + pure = "on", + visibility = ["//visibility:public"], +) + +load("@io_bazel_rules_docker//go:image.bzl", "go_image") + +go_image( + name = "image", + binary = ":helloworld", +) + +load("@k8s_object//:defaults.bzl", "k8s_object") + +k8s_object( + name = "service", + images = { + "sample:latest": ":image", + }, + template = ":service.yaml", +) + +k8s_object( + name = "updated_service", + images = { + "sample:latest": ":image", + }, + template = ":updated_service.yaml", +) + +k8s_object( + name = "pinned_service", + images = { + "sample:latest": ":image", + }, + template = ":pinned_service.yaml", +) + +load("@io_bazel_rules_k8s//k8s:objects.bzl", "k8s_objects") + +k8s_objects( + name = "everything", + objects = [ + ":service", + ], +) + +k8s_objects( + name = "updated_everything", + objects = [ + ":updated_service", + ], +) + +k8s_objects( + name = "pinned_everything", + objects = [ + ":pinned_service", + ], +) diff --git a/sample/service/README.md b/sample/service/README.md new file mode 100644 index 000000000000..91e2ed4dcffd --- /dev/null +++ b/sample/service/README.md @@ -0,0 +1,124 @@ +# Service + +A simple web server that you can use for testing Service resource. +It reads in an env variable 'TARGET' and prints "Hello World: ${TARGET}!" if +TARGET is not specified, it will use "NOT SPECIFIED" as the TARGET. + +## Prerequisites + +1. [Setup your development environment](../../DEVELOPMENT.md#getting-started) +2. [Start Elafros](../../README.md#start-elafros) + +## Running + +You can deploy this to Elafros from the root directory via: +```shell +bazel run sample/service:everything.create +``` + +Once deployed, you can inspect the created resources with `kubectl` commands: + +```shell +# This will show the service that we created: +kubectl get service.elafros.dev -oyaml +``` + +```shell +# This will show the route that was created by the service: +kubectl get route -o yaml +``` + +```shell +# This will show the configuration that was created by the service: +kubectl get configurations -o yaml +``` + +```shell +# This will show the Revision that was created the configuration created by the service: +kubectl get revisions -o yaml + +``` + +To access this service via `curl`, we first need to determine its ingress address: +```shell +watch kubectl get ingress +``` + +When the ingress is ready, you'll see an IP address in the ADDRESS field: + +``` +NAME HOSTS ADDRESS PORTS AGE +route-example-ela-ingress demo.myhost.net 80 14s +``` + +Once the `ADDRESS` gets assigned to the cluster, you can run: + +```shell +# Put the Ingress Host name into an environment variable. +export SERVICE_HOST=`kubectl get route route-example -o jsonpath="{.status.domain}"` + +# Put the Ingress IP into an environment variable. +export SERVICE_IP=`kubectl get ingress route-example-ela-ingress -o jsonpath="{.status.loadBalancer.ingress[*]['ip']}"` +``` + +If your cluster is running outside a cloud provider (for example on Minikube), +your ingress will never get an address. In that case, use the istio `hostIP` and `nodePort` as the service IP: + +```shell +export SERVICE_IP=$(kubectl get po -l istio=ingress -n istio-system -o 'jsonpath={.items[0].status.hostIP}'):$(kubectl get svc istio-ingress -n istio-system -o 'jsonpath={.spec.ports[?(@.port==80)].nodePort}') +``` + +Now curl the service IP as if DNS were properly configured: + +```shell +curl --header "Host:$SERVICE_HOST" http://${SERVICE_IP} +# Hello World: shiniestnewestversion! +``` + +## Updating + +You can update this to a new version. For example, update it with a new configuration.yaml via: +```shell +bazel run sample/service:updated_everything.apply +``` + +Once deployed, traffic will shift to the new revision automatically. You can verify the new version +by checking route status: +```shell +# This will show the route that we created: +kubectl get route -o yaml +``` + +Or curling the service: +```shell +curl --header "Host:$SERVICE_HOST" http://${SERVICE_IP} +# Hello World: evenshinierversion! +``` + +## Pinning a service + +You can pin a Service to a specific revision. For example, update it with a new service.yaml via: +```shell +bazel run sample/service:pinned_everything.apply +``` + +Once deployed, traffic will shift to the previous (first) revision automatically. You can verify the new version +by checking route status: +```shell +# This will show the route that we created: +kubectl get route -o yaml +``` + +Or curling the service: +```shell +curl --header "Host:$SERVICE_HOST" http://${SERVICE_IP} +# Hello World: shiniestnewestversion! +``` + +## Cleaning up + +To clean up the sample service: + +```shell +bazel run sample/service:everything.delete +``` diff --git a/sample/service/helloworld.go b/sample/service/helloworld.go new file mode 100644 index 000000000000..e3948aeb3fff --- /dev/null +++ b/sample/service/helloworld.go @@ -0,0 +1,42 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "log" + "net/http" + "os" +) + +func handler(w http.ResponseWriter, r *http.Request) { + log.Print("Hello world received a request.") + target := os.Getenv("TARGET") + if target == "" { + target = "NOT SPECIFIED" + } + fmt.Fprintf(w, "Hello World: %s!\n", target) +} + +func main() { + flag.Parse() + log.Print("Hello world sample started.") + + http.HandleFunc("/", handler) + http.ListenAndServe(":8080", nil) +} diff --git a/sample/service/pinned_service.yaml b/sample/service/pinned_service.yaml new file mode 100644 index 000000000000..4e49fb2875e5 --- /dev/null +++ b/sample/service/pinned_service.yaml @@ -0,0 +1,36 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: elafros.dev/v1alpha1 +kind: Service +metadata: + name: service-example + namespace: default +spec: + pinned: + revisionName: service-example-00001 + configuration: + revisionTemplate: + spec: + container: + image: sample:latest + env: + - name: TARGET + value: evenshinierversionbutnotshownautomatically + readinessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 3 + periodSeconds: 3 diff --git a/sample/service/service.yaml b/sample/service/service.yaml new file mode 100644 index 000000000000..f6ec9d777831 --- /dev/null +++ b/sample/service/service.yaml @@ -0,0 +1,35 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: elafros.dev/v1alpha1 +kind: Service +metadata: + name: service-example + namespace: default +spec: + runLatest: + configuration: + revisionTemplate: + spec: + container: + image: sample:latest + env: + - name: TARGET + value: shiniestnewestversion + readinessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 3 + periodSeconds: 3 diff --git a/sample/service/updated_service.yaml b/sample/service/updated_service.yaml new file mode 100644 index 000000000000..d5d5bdabfb0c --- /dev/null +++ b/sample/service/updated_service.yaml @@ -0,0 +1,35 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: elafros.dev/v1alpha1 +kind: Service +metadata: + name: service-example + namespace: default +spec: + runLatest: + configuration: + revisionTemplate: + spec: + container: + image: sample:latest + env: + - name: TARGET + value: evenshinierversion + readinessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 3 + periodSeconds: 3 From 07bf015265c9b860461c289e35c3dc23623bcc48 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Fri, 6 Apr 2018 12:14:39 -0700 Subject: [PATCH 05/12] Apply Service label to route / configuration. add tests --- pkg/controller/service/ela_resource.go | 18 +++ .../service/service_configuration_test.go | 119 ++++++++++++++++++ pkg/controller/service/service_route_test.go | 72 +++++++++++ sample/service/BUILD | 2 +- 4 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 pkg/controller/service/ela_resource.go create mode 100644 pkg/controller/service/service_configuration_test.go create mode 100644 pkg/controller/service/service_route_test.go diff --git a/pkg/controller/service/ela_resource.go b/pkg/controller/service/ela_resource.go new file mode 100644 index 000000000000..06e004f9c374 --- /dev/null +++ b/pkg/controller/service/ela_resource.go @@ -0,0 +1,18 @@ +package service + +import ( + "github.com/elafros/elafros/pkg/apis/ela" + "github.com/elafros/elafros/pkg/apis/ela/v1alpha1" +) + +// MakeElaResourceLabels constructs the labels we will apply to Route and Configuration +// resources. +func MakeElaResourceLabels(s *v1alpha1.Service) map[string]string { + labels := make(map[string]string, len(s.ObjectMeta.Labels)+1) + labels[ela.ServiceLabelKey] = s.Name + + for k, v := range s.ObjectMeta.Labels { + labels[k] = v + } + return labels +} diff --git a/pkg/controller/service/service_configuration_test.go b/pkg/controller/service/service_configuration_test.go new file mode 100644 index 000000000000..820fd2e07c46 --- /dev/null +++ b/pkg/controller/service/service_configuration_test.go @@ -0,0 +1,119 @@ +/* +Copyright 2018 Google LLC +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "testing" + + "github.com/elafros/elafros/pkg/apis/ela/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + testServiceName string = "test-service" + testServiceNamespace string = "test-service-namespace" + testRevisionName string = "test-revision-name" + testContainerNameRunLatest string = "test-container-run-latest" + testContainerNamePinned string = "test-container-pinned" +) + +func createConfiguration(containerName string) v1alpha1.ConfigurationSpec { + return v1alpha1.ConfigurationSpec{ + RevisionTemplate: v1alpha1.RevisionTemplateSpec{ + Spec: v1alpha1.RevisionSpec{ + Container: &corev1.Container{ + Name: containerName, + }, + }, + }, + } +} + +func createServiceMeta() *v1alpha1.Service { + return &v1alpha1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: testServiceName, + Namespace: testServiceNamespace, + }, + } +} + +func createServiceWithRunLatest() *v1alpha1.Service { + s := createServiceMeta() + s.Spec = v1alpha1.ServiceSpec{ + RunLatest: &v1alpha1.RunLatestType{ + Configuration: createConfiguration(testContainerNameRunLatest), + }, + } + return s +} + +func createServiceWithPinned() *v1alpha1.Service { + s := createServiceMeta() + s.Spec = v1alpha1.ServiceSpec{ + Pinned: &v1alpha1.PinnedType{ + RevisionName: testRevisionName, + Configuration: createConfiguration(testContainerNamePinned), + }, + } + return s +} + +func TestRunLatest(t *testing.T) { + s := createServiceWithRunLatest() + c := MakeServiceConfiguration(s) + if got, want := c.Name, testServiceName; got != want { + t.Errorf("expected %q for service name got %q", want, got) + } + if got, want := c.Namespace, testServiceNamespace; got != want { + t.Errorf("expected %q for service namespace got %q", want, got) + } + if got, want := c.Spec.RevisionTemplate.Spec.Container.Name, testContainerNameRunLatest; got != want { + t.Errorf("expected %q for container name got %q", want, got) + } + expectOwnerReferencesSetCorrectly(t, c.OwnerReferences) +} + +func TestPinned(t *testing.T) { + s := createServiceWithPinned() + c := MakeServiceConfiguration(s) + if got, want := c.Name, testServiceName; got != want { + t.Errorf("expected %q for service name got %q", want, got) + } + if got, want := c.Namespace, testServiceNamespace; got != want { + t.Errorf("expected %q for service namespace got %q", want, got) + } + if got, want := c.Spec.RevisionTemplate.Spec.Container.Name, testContainerNamePinned; got != want { + t.Errorf("expected %q for container name got %q", want, got) + } + expectOwnerReferencesSetCorrectly(t, c.OwnerReferences) +} + +func expectOwnerReferencesSetCorrectly(t *testing.T, ownerRefs []metav1.OwnerReference) { + if got, want := len(ownerRefs), 1; got != want { + t.Errorf("expected %d owner refs got %d", want, got) + return + } + or := ownerRefs[0] + if got, want := or.Name, testServiceName; got != want { + t.Errorf("expected %q owner refs name got %q", want, got) + } + if got, want := or.Kind, controllerKind.Kind; got != want { + t.Errorf("expected %q owner refs kind got %q", want, got) + } + if got, want := or.APIVersion, controllerKind.GroupVersion().String(); got != want { + t.Errorf("expected %q owner refs kind got %q", want, got) + } +} diff --git a/pkg/controller/service/service_route_test.go b/pkg/controller/service/service_route_test.go new file mode 100644 index 000000000000..5cce0a0ba34c --- /dev/null +++ b/pkg/controller/service/service_route_test.go @@ -0,0 +1,72 @@ +/* +Copyright 2018 Google LLC +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "testing" +) + +const ( + testConfigName string = "test-configuration" +) + +func TestRouteRunLatest(t *testing.T) { + s := createServiceWithRunLatest() + r := MakeServiceRoute(s, testConfigName, "") + if got, want := r.Name, testServiceName; got != want { + t.Errorf("expected %q for service name got %q", want, got) + } + if got, want := r.Namespace, testServiceNamespace; got != want { + t.Errorf("expected %q for service namespace got %q", want, got) + } + if got, want := len(r.Spec.Traffic), 1; got != want { + t.Fatalf("expected %d traffic targets got %d", want, got) + } + tt := r.Spec.Traffic[0] + if got, want := tt.Percent, 100; got != want { + t.Errorf("expected %d percent got %d", want, got) + } + if got, want := tt.RevisionName, ""; got != want { + t.Errorf("expected %q revisionName got %q", want, got) + } + if got, want := tt.ConfigurationName, testConfigName; got != want { + t.Errorf("expected %q configurationname got %q", want, got) + } + expectOwnerReferencesSetCorrectly(t, r.OwnerReferences) +} + +func TestRoutePinned(t *testing.T) { + s := createServiceWithRunLatest() + r := MakeServiceRoute(s, testConfigName, testRevisionName) + if got, want := r.Name, testServiceName; got != want { + t.Errorf("expected %q for service name got %q", want, got) + } + if got, want := r.Namespace, testServiceNamespace; got != want { + t.Errorf("expected %q for service namespace got %q", want, got) + } + if got, want := len(r.Spec.Traffic), 1; got != want { + t.Fatalf("expected %d traffic targets, got %d", want, got) + } + tt := r.Spec.Traffic[0] + if got, want := tt.Percent, 100; got != want { + t.Errorf("expected %d percent got %d", want, got) + } + if got, want := tt.RevisionName, testRevisionName; got != want { + t.Errorf("expected %q revisionName got %q", want, got) + } + if got, want := tt.ConfigurationName, ""; got != want { + t.Errorf("expected %q configurationname got %q", want, got) + } + expectOwnerReferencesSetCorrectly(t, r.OwnerReferences) +} diff --git a/sample/service/BUILD b/sample/service/BUILD index dd6769577bc2..17db276c4d99 100644 --- a/sample/service/BUILD +++ b/sample/service/BUILD @@ -3,7 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") go_library( name = "go_default_library", srcs = ["helloworld.go"], - importpath = "github.com/elafros/elafros/sample/helloworld", + importpath = "github.com/elafros/elafros/sample/service", visibility = ["//visibility:private"], ) From 2b3ea90bba15c1097beeaadb73313682c138a9de Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Fri, 6 Apr 2018 12:26:10 -0700 Subject: [PATCH 06/12] stragglers --- pkg/apis/ela/register.go | 6 +++- pkg/controller/service/BUILD.bazel | 10 +++++++ .../service/service_configuration.go | 1 + .../service/service_configuration_test.go | 28 +++++++++++++++++++ pkg/controller/service/service_route.go | 1 + pkg/controller/service/service_route_test.go | 22 +++++++++++++++ 6 files changed, 67 insertions(+), 1 deletion(-) diff --git a/pkg/apis/ela/register.go b/pkg/apis/ela/register.go index 820ac4116840..a8d415bde265 100644 --- a/pkg/apis/ela/register.go +++ b/pkg/apis/ela/register.go @@ -19,7 +19,7 @@ package ela const ( GroupName = "elafros.dev" - // ConfigurationLabelKey is the label key attached to a Revison indicating by + // ConfigurationLabelKey is the label key attached to a Revision indicating by // which Configuration it is created. ConfigurationLabelKey = GroupName + "/configuration" @@ -38,4 +38,8 @@ const ( // AutoscalerLabelKey is the label key attached to a autoscaler pod indicating by // which Autoscaler deployment it is created. AutoscalerLabelKey = GroupName + "/autoscaler" + + // ServiceLabelKey is the label key attached to a Route and Configuration indicating by + // which Service they are created. + ServiceLabelKey = GroupName + "/service" ) diff --git a/pkg/controller/service/BUILD.bazel b/pkg/controller/service/BUILD.bazel index 98b51ca25acf..9ac59edf9846 100644 --- a/pkg/controller/service/BUILD.bazel +++ b/pkg/controller/service/BUILD.bazel @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = [ + "ela_resource.go", "service.go", "service_configuration.go", "service_route.go", @@ -10,6 +11,7 @@ go_library( importpath = "github.com/elafros/elafros/pkg/controller/service", visibility = ["//visibility:public"], deps = [ + "//pkg/apis/ela:go_default_library", "//pkg/apis/ela/v1alpha1:go_default_library", "//pkg/client/clientset/versioned:go_default_library", "//pkg/client/informers/externalversions:go_default_library", @@ -36,7 +38,15 @@ go_library( go_test( name = "go_default_test", srcs = [ + "service_configuration_test.go", + "service_route_test.go", "service_test.go", ], embed = [":go_default_library"], + deps = [ + "//pkg/apis/ela:go_default_library", + "//pkg/apis/ela/v1alpha1:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + ], ) diff --git a/pkg/controller/service/service_configuration.go b/pkg/controller/service/service_configuration.go index 0f44aaa2560a..6982bcfb81f1 100644 --- a/pkg/controller/service/service_configuration.go +++ b/pkg/controller/service/service_configuration.go @@ -31,6 +31,7 @@ func MakeServiceConfiguration(service *v1alpha1.Service) *v1alpha1.Configuration OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(service, controllerKind), }, + Labels: MakeElaResourceLabels(service), }, } diff --git a/pkg/controller/service/service_configuration_test.go b/pkg/controller/service/service_configuration_test.go index 820fd2e07c46..3653d69caa42 100644 --- a/pkg/controller/service/service_configuration_test.go +++ b/pkg/controller/service/service_configuration_test.go @@ -16,6 +16,7 @@ package service import ( "testing" + "github.com/elafros/elafros/pkg/apis/ela" "github.com/elafros/elafros/pkg/apis/ela/v1alpha1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -27,6 +28,9 @@ const ( testRevisionName string = "test-revision-name" testContainerNameRunLatest string = "test-container-run-latest" testContainerNamePinned string = "test-container-pinned" + testLabelKey string = "test-label-key" + testLabelValuePinned string = "test-label-value-pinned" + testLabelValueRunLatest string = "test-label-value-run-latest" ) func createConfiguration(containerName string) v1alpha1.ConfigurationSpec { @@ -57,6 +61,8 @@ func createServiceWithRunLatest() *v1alpha1.Service { Configuration: createConfiguration(testContainerNameRunLatest), }, } + s.Labels = make(map[string]string, 2) + s.Labels[testLabelKey] = testLabelValueRunLatest return s } @@ -68,6 +74,8 @@ func createServiceWithPinned() *v1alpha1.Service { Configuration: createConfiguration(testContainerNamePinned), }, } + s.Labels = make(map[string]string, 2) + s.Labels[testLabelKey] = testLabelValuePinned return s } @@ -84,6 +92,16 @@ func TestRunLatest(t *testing.T) { t.Errorf("expected %q for container name got %q", want, got) } expectOwnerReferencesSetCorrectly(t, c.OwnerReferences) + + if got, want := len(c.Labels), 2; got != want { + t.Errorf("expected %d labels got %d", want, got) + } + if got, want := c.Labels[testLabelKey], testLabelValueRunLatest; got != want { + t.Errorf("expected %q labels got %q", want, got) + } + if got, want := c.Labels[ela.ServiceLabelKey], testServiceName; got != want { + t.Errorf("expected %q labels got %q", want, got) + } } func TestPinned(t *testing.T) { @@ -99,6 +117,16 @@ func TestPinned(t *testing.T) { t.Errorf("expected %q for container name got %q", want, got) } expectOwnerReferencesSetCorrectly(t, c.OwnerReferences) + + if got, want := len(c.Labels), 2; got != want { + t.Errorf("expected %d labels got %d", want, got) + } + if got, want := c.Labels[testLabelKey], testLabelValuePinned; got != want { + t.Errorf("expected %q labels got %q", want, got) + } + if got, want := c.Labels[ela.ServiceLabelKey], testServiceName; got != want { + t.Errorf("expected %q labels got %q", want, got) + } } func expectOwnerReferencesSetCorrectly(t *testing.T, ownerRefs []metav1.OwnerReference) { diff --git a/pkg/controller/service/service_route.go b/pkg/controller/service/service_route.go index e0ab24519952..53085472d8b8 100644 --- a/pkg/controller/service/service_route.go +++ b/pkg/controller/service/service_route.go @@ -31,6 +31,7 @@ func MakeServiceRoute(service *v1alpha1.Service, configName string, revisionName OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(service, controllerKind), }, + Labels: MakeElaResourceLabels(service), }, } diff --git a/pkg/controller/service/service_route_test.go b/pkg/controller/service/service_route_test.go index 5cce0a0ba34c..70ecf1019ab9 100644 --- a/pkg/controller/service/service_route_test.go +++ b/pkg/controller/service/service_route_test.go @@ -15,6 +15,8 @@ package service import ( "testing" + + "github.com/elafros/elafros/pkg/apis/ela" ) const ( @@ -44,6 +46,16 @@ func TestRouteRunLatest(t *testing.T) { t.Errorf("expected %q configurationname got %q", want, got) } expectOwnerReferencesSetCorrectly(t, r.OwnerReferences) + + if got, want := len(r.Labels), 2; got != want { + t.Errorf("expected %d labels got %d", want, got) + } + if got, want := r.Labels[testLabelKey], testLabelValueRunLatest; got != want { + t.Errorf("expected %q labels got %q", want, got) + } + if got, want := r.Labels[ela.ServiceLabelKey], testServiceName; got != want { + t.Errorf("expected %q labels got %q", want, got) + } } func TestRoutePinned(t *testing.T) { @@ -69,4 +81,14 @@ func TestRoutePinned(t *testing.T) { t.Errorf("expected %q configurationname got %q", want, got) } expectOwnerReferencesSetCorrectly(t, r.OwnerReferences) + + if got, want := len(r.Labels), 2; got != want { + t.Errorf("expected %d labels got %d", want, got) + } + if got, want := r.Labels[testLabelKey], testLabelValueRunLatest; got != want { + t.Errorf("expected %q labels got %q", want, got) + } + if got, want := r.Labels[ela.ServiceLabelKey], testServiceName; got != want { + t.Errorf("expected %q labels got %q", want, got) + } } From e0cf0a419d0d6896ab9a582d0289121a5f6cf706 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Fri, 6 Apr 2018 15:00:28 -0700 Subject: [PATCH 07/12] address code review feedback --- pkg/controller/service/service.go | 24 ++++++++------------ pkg/controller/service/service_route.go | 10 ++++---- pkg/controller/service/service_route_test.go | 4 ++-- pkg/controller/service/service_test.go | 1 + pkg/queue/BUILD.bazel | 2 +- sample/service/README.md | 6 ++--- 6 files changed, 23 insertions(+), 24 deletions(-) diff --git a/pkg/controller/service/service.go b/pkg/controller/service/service.go index 9fac635815a1..7f80f94a053c 100644 --- a/pkg/controller/service/service.go +++ b/pkg/controller/service/service.go @@ -45,8 +45,8 @@ import ( ) var ( - controllerKind = v1alpha1.SchemeGroupVersion.WithKind("Service") - routeProcessItemCount = prometheus.NewCounterVec(prometheus.CounterOpts{ + controllerKind = v1alpha1.SchemeGroupVersion.WithKind("Service") + serviceProcessItemCount = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "elafros", Name: "service_process_item_count", Help: "Counter to keep track of items in the service work queue", @@ -80,7 +80,7 @@ type Controller struct { } func init() { - prometheus.MustRegister(routeProcessItemCount) + prometheus.MustRegister(serviceProcessItemCount) } // NewController initializes the controller and is called by the generated code @@ -96,7 +96,7 @@ func NewController( glog.Infof("Service controller Init") - // obtain references to a shared index informer for the Services and + // obtain references to a shared index informer for the Services. informer := elaInformerFactory.Elafros().V1alpha1().Services() // Create event broadcaster @@ -211,7 +211,7 @@ func (c *Controller) processNextWorkItem() bool { return nil, controller.PromLabelValueSuccess }(obj) - routeProcessItemCount.With(prometheus.Labels{"status": promStatus}).Inc() + serviceProcessItemCount.With(prometheus.Labels{"status": promStatus}).Inc() if err != nil { runtime.HandleError(err) @@ -274,12 +274,10 @@ func (c *Controller) updateServiceEvent(key string) error { // TODO: If revision is specified, check that the revision is ready before // switching routes to it. Though route controller might just do the right thing? - revisionName := "" - if service.Spec.Pinned != nil { - revisionName = service.Spec.Pinned.RevisionName - } - route := MakeServiceRoute(service, config.Name, revisionName) + route := MakeServiceRoute(service, config.Name) return c.reconcileRoute(route) + + // TODO: update status appropriately. } func (c *Controller) updateStatus(service *v1alpha1.Service) (*v1alpha1.Service, error) { @@ -307,8 +305,7 @@ func (c *Controller) reconcileConfiguration(config *v1alpha1.Configuration) erro return err } } - config.SetGeneration(existing.GetGeneration()) - + // TODO(vaikas): Perhaps only update if there are actual changes. copy := existing.DeepCopy() copy.Spec = config.Spec _, err = configClient.Update(copy) @@ -325,8 +322,7 @@ func (c *Controller) reconcileRoute(route *v1alpha1.Route) error { return err } } - route.SetGeneration(existing.GetGeneration()) - + // TODO(vaikas): Perhaps only update if there are actual changes. copy := existing.DeepCopy() copy.Spec = route.Spec _, err = routeClient.Update(copy) diff --git a/pkg/controller/service/service_route.go b/pkg/controller/service/service_route.go index 53085472d8b8..a8f3613f0cba 100644 --- a/pkg/controller/service/service_route.go +++ b/pkg/controller/service/service_route.go @@ -23,7 +23,7 @@ import ( ) // MakeServiceRoute creates a Route from a Service object. -func MakeServiceRoute(service *v1alpha1.Service, configName string, revisionName string) *v1alpha1.Route { +func MakeServiceRoute(service *v1alpha1.Service, configName string) *v1alpha1.Route { c := &v1alpha1.Route{ ObjectMeta: metav1.ObjectMeta{ Name: service.Name, @@ -38,10 +38,12 @@ func MakeServiceRoute(service *v1alpha1.Service, configName string, revisionName tt := v1alpha1.TrafficTarget{ Percent: 100, } - if len(revisionName) != 0 { - tt.RevisionName = revisionName - } else { + // If there's RunLatest, use the configName, otherwise pin to a specific Revision + // as specified in the Pinned section of the Service spec. + if service.Spec.RunLatest != nil { tt.ConfigurationName = configName + } else { + tt.RevisionName = service.Spec.Pinned.RevisionName } c.Spec.Traffic = append(c.Spec.Traffic, tt) return c diff --git a/pkg/controller/service/service_route_test.go b/pkg/controller/service/service_route_test.go index 70ecf1019ab9..eb98641c1376 100644 --- a/pkg/controller/service/service_route_test.go +++ b/pkg/controller/service/service_route_test.go @@ -25,7 +25,7 @@ const ( func TestRouteRunLatest(t *testing.T) { s := createServiceWithRunLatest() - r := MakeServiceRoute(s, testConfigName, "") + r := MakeServiceRoute(s, testConfigName) if got, want := r.Name, testServiceName; got != want { t.Errorf("expected %q for service name got %q", want, got) } @@ -60,7 +60,7 @@ func TestRouteRunLatest(t *testing.T) { func TestRoutePinned(t *testing.T) { s := createServiceWithRunLatest() - r := MakeServiceRoute(s, testConfigName, testRevisionName) + r := MakeServiceRoute(s, testConfigName) if got, want := r.Name, testServiceName; got != want { t.Errorf("expected %q for service name got %q", want, got) } diff --git a/pkg/controller/service/service_test.go b/pkg/controller/service/service_test.go index dae7f5ed9371..2f22f856ef18 100644 --- a/pkg/controller/service/service_test.go +++ b/pkg/controller/service/service_test.go @@ -18,5 +18,6 @@ import ( ) func testNothing(t *testing.T) { + // TODO(vaikas): Implement controller tests return } diff --git a/pkg/queue/BUILD.bazel b/pkg/queue/BUILD.bazel index 27c77516597c..6f488b5256e8 100644 --- a/pkg/queue/BUILD.bazel +++ b/pkg/queue/BUILD.bazel @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") +load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", diff --git a/sample/service/README.md b/sample/service/README.md index 91e2ed4dcffd..9a043792ae22 100644 --- a/sample/service/README.md +++ b/sample/service/README.md @@ -48,17 +48,17 @@ When the ingress is ready, you'll see an IP address in the ADDRESS field: ``` NAME HOSTS ADDRESS PORTS AGE -route-example-ela-ingress demo.myhost.net 80 14s +service-example-ela-ingress demo.myhost.net 80 14s ``` Once the `ADDRESS` gets assigned to the cluster, you can run: ```shell # Put the Ingress Host name into an environment variable. -export SERVICE_HOST=`kubectl get route route-example -o jsonpath="{.status.domain}"` +export SERVICE_HOST=`kubectl get route service-example -o jsonpath="{.status.domain}"` # Put the Ingress IP into an environment variable. -export SERVICE_IP=`kubectl get ingress route-example-ela-ingress -o jsonpath="{.status.loadBalancer.ingress[*]['ip']}"` +export SERVICE_IP=`kubectl get ingress service-example-ela-ingress -o jsonpath="{.status.loadBalancer.ingress[*]['ip']}"` ``` If your cluster is running outside a cloud provider (for example on Minikube), From e799cca8d01d36f1e6420752de668a1d27fa79ac Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Fri, 6 Apr 2018 15:08:32 -0700 Subject: [PATCH 08/12] remove changes to pkg/queue/BUILD.bazel --- pkg/queue/BUILD.bazel | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/pkg/queue/BUILD.bazel b/pkg/queue/BUILD.bazel index 6f488b5256e8..4e29bba9446b 100644 --- a/pkg/queue/BUILD.bazel +++ b/pkg/queue/BUILD.bazel @@ -7,13 +7,3 @@ go_library( visibility = ["//visibility:public"], deps = ["//pkg/autoscaler:go_default_library"], ) - -go_test( - name = "go_default_test", - srcs = ["queue_test.go"], - embed = [":go_default_library"], - deps = [ - "//pkg/autoscaler:go_default_library", - "//vendor/github.com/google/go-cmp/cmp:go_default_library", - ], -) From b711ed179b9bb68b19383fd7d47e8a611395df2a Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Fri, 6 Apr 2018 15:12:43 -0700 Subject: [PATCH 09/12] fix tests --- pkg/controller/service/service_route_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/service/service_route_test.go b/pkg/controller/service/service_route_test.go index eb98641c1376..dd9fa5a76da4 100644 --- a/pkg/controller/service/service_route_test.go +++ b/pkg/controller/service/service_route_test.go @@ -59,7 +59,7 @@ func TestRouteRunLatest(t *testing.T) { } func TestRoutePinned(t *testing.T) { - s := createServiceWithRunLatest() + s := createServiceWithPinned() r := MakeServiceRoute(s, testConfigName) if got, want := r.Name, testServiceName; got != want { t.Errorf("expected %q for service name got %q", want, got) @@ -85,7 +85,7 @@ func TestRoutePinned(t *testing.T) { if got, want := len(r.Labels), 2; got != want { t.Errorf("expected %d labels got %d", want, got) } - if got, want := r.Labels[testLabelKey], testLabelValueRunLatest; got != want { + if got, want := r.Labels[testLabelKey], testLabelValuePinned; got != want { t.Errorf("expected %q labels got %q", want, got) } if got, want := r.Labels[ela.ServiceLabelKey], testServiceName; got != want { From 1c0d9b51b3a19cdeac1215ca1d6b07c1c42e3825 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Fri, 6 Apr 2018 16:50:31 -0700 Subject: [PATCH 10/12] address review comments --- pkg/controller/service/service.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/controller/service/service.go b/pkg/controller/service/service.go index 7f80f94a053c..e535a508e46a 100644 --- a/pkg/controller/service/service.go +++ b/pkg/controller/service/service.go @@ -85,7 +85,6 @@ func init() { // NewController initializes the controller and is called by the generated code // Registers eventhandlers to enqueue events -//TODO(vaikas): somewhat generic (generic behavior) func NewController( kubeclientset kubernetes.Interface, elaclientset clientset.Interface, @@ -130,7 +129,6 @@ func NewController( // as syncing informer caches and starting workers. It will block until stopCh // is closed, at which point it will shutdown the workqueue and wait for // workers to finish processing their current work items. -//TODO(grantr): generic func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) error { defer runtime.HandleCrash() defer c.workqueue.ShutDown() @@ -145,7 +143,7 @@ func (c *Controller) Run(threadiness int, stopCh <-chan struct{}) error { } glog.Info("Starting workers") - // Launch two workers to process Service resources + // Launch threadiness workers to process Service resources for i := 0; i < threadiness; i++ { go wait.Until(c.runWorker, time.Second, stopCh) } @@ -167,7 +165,6 @@ func (c *Controller) runWorker() { // processNextWorkItem will read a single work item off the workqueue and // attempt to process it, by calling the updateServiceEvent. -//TODO(vaikas): generic func (c *Controller) processNextWorkItem() bool { obj, shutdown := c.workqueue.Get() @@ -304,6 +301,7 @@ func (c *Controller) reconcileConfiguration(config *v1alpha1.Configuration) erro _, err := configClient.Create(config) return err } + return err } // TODO(vaikas): Perhaps only update if there are actual changes. copy := existing.DeepCopy() @@ -321,6 +319,7 @@ func (c *Controller) reconcileRoute(route *v1alpha1.Route) error { _, err := routeClient.Create(route) return err } + return err } // TODO(vaikas): Perhaps only update if there are actual changes. copy := existing.DeepCopy() From 0ee3c940d220634a183b56bb7ea7b73fbe3e4b50 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Fri, 6 Apr 2018 17:05:24 -0700 Subject: [PATCH 11/12] more code review comments --- pkg/controller/service/service.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/controller/service/service.go b/pkg/controller/service/service.go index e535a508e46a..3bca350a98a4 100644 --- a/pkg/controller/service/service.go +++ b/pkg/controller/service/service.go @@ -222,7 +222,8 @@ func (c *Controller) processNextWorkItem() bool { // converts it into a namespace/name string which is then put onto the work // queue. This method should *not* be passed resources of any type other than // Service. -//TODO(grantr): generic +// TODO(grantr): generic +// TODO: assert the object passed in is of the correct type. func (c *Controller) enqueueService(obj interface{}) { var key string var err error @@ -246,10 +247,6 @@ func (c *Controller) updateServiceEvent(key string) error { // Get the Service resource with this namespace/name service, err := c.lister.Services(namespace).Get(name) - - // Don't modify the informers copy - service = service.DeepCopy() - if err != nil { // The resource may no longer exist, in which case we stop // processing. @@ -261,6 +258,9 @@ func (c *Controller) updateServiceEvent(key string) error { return err } + // Don't modify the informers copy + service = service.DeepCopy() + glog.Infof("Running reconcile Service for %s\n%+v\n", service.Name, service) config := MakeServiceConfiguration(service) From 3f14783639bdd237b993ec71af6f91bbd5948f03 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Mon, 9 Apr 2018 11:21:37 -0400 Subject: [PATCH 12/12] Address code review comment --- pkg/controller/service/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/service/service.go b/pkg/controller/service/service.go index 3bca350a98a4..148d96e1516b 100644 --- a/pkg/controller/service/service.go +++ b/pkg/controller/service/service.go @@ -264,7 +264,7 @@ func (c *Controller) updateServiceEvent(key string) error { glog.Infof("Running reconcile Service for %s\n%+v\n", service.Name, service) config := MakeServiceConfiguration(service) - if err = c.reconcileConfiguration(config); err != nil { + if err := c.reconcileConfiguration(config); err != nil { return err }