Merge pull request #945 from cliveseldon/bypass_engine

Bypass engine via annotation

doc/source/graph/svcorch.md

@@ -4,7 +4,7 @@ The service orchestrator is a component that is added to your inference graph to
 
  * Correctly manage the request/response paths described by your inference graph
  * Expose prometheus metrics
-
+ * Add meta data to the response
 
 ## Resource Requests/Limits for Service Orchetsrator
 
@@ -67,4 +67,40 @@ The service orchestrator is a Java component. You can directly control its java
 You can manipulate some of the functionality of the service orchestrator by adding specific environment variables to the `svcOrchSpec` section.
 
  * [Configure Jaeger Tracing Example](../graph/distributed-tracing.html)
- * [Set logging level in service orchestrator engine](../analytics/log_level.html#setting-log-level-in-the-seldon-engine)
+ * [Set logging level in service orchestrator engine](../analytics/log_level.html#setting-log-level-in-the-seldon-engine)
+
+## Bypass Service Orchestrator (version >= 0.5.0, alpha feature)
+
+If you are deploying a single model then for those wishing to minimize the network hops and have lowest latency and resource usage for their deployed model you can opt out of having the service orchestrator included. To do this add the annotation `seldon.io/no-engine: "true"` to the predictor. The predictor must contain just a single node graph. An example is shown below:
+
+
+```YAML
+apiVersion: machinelearning.seldon.io/v1alpha2
+kind: SeldonDeployment
+metadata:
+  labels:
+    app: seldon
+  name: noengine
+spec:
+  name: noeng
+  predictors:
+  - annotations:
+      seldon.io/no-engine: "true"
+    componentSpecs:
+    - spec:
+        containers:
+        - image: seldonio/mock_classifier_rest:1.3
+          name: classifier
+    graph:
+      children: []
+      endpoint:
+        type: REST
+      name: classifier
+      type: MODEL
+    name: noeng
+    replicas: 1
+```
+
+In these cases the external API requests will be sent directly to your model. At present only the python wrapper (>=0.13-SNAPSHOT) has been modified to allow this.
+
+Note no metrics or extra data will be added to the request so this would need to be done by your model itself if needed. 

examples/models/mean_classifier/Makefile

@@ -2,13 +2,13 @@ VERSION=1.3
 IMAGE_BASE=seldonio/mock_classifier
 
 build_rest:
-	s2i build -E environment_rest . seldonio/seldon-core-s2i-python36:0.11-SNAPSHOT ${IMAGE_BASE}_rest:${VERSION}
+	s2i build -E environment_rest . seldonio/seldon-core-s2i-python36:0.13-SNAPSHOT ${IMAGE_BASE}_rest:${VERSION}
 
 push_rest:
 	docker push ${IMAGE_BASE}_rest:${VERSION}
 
 build_grpc:
-	s2i build -E environment_grpc . seldonio/seldon-core-s2i-python36:0.11-SNAPSHOT ${IMAGE_BASE}_grpc:${VERSION}
+	s2i build -E environment_grpc . seldonio/seldon-core-s2i-python36:0.13-SNAPSHOT ${IMAGE_BASE}_grpc:${VERSION}
 
 push_grpc:
 	docker push ${IMAGE_BASE}_grpc:${VERSION}

notebooks/resources/model_no_engine.yaml

@@ -0,0 +1,25 @@
+apiVersion: machinelearning.seldon.io/v1alpha2
+kind: SeldonDeployment
+metadata:
+  labels:
+    app: seldon
+  name: noengine
+spec:
+  name: noeng
+  predictors:
+  - annotations:
+      seldon.io/no-engine: "true"
+    componentSpecs:
+    - spec:
+        containers:
+        - image: seldonio/mock_classifier_rest:1.3
+          imagePullPolicy: Never
+          name: classifier
+    graph:
+      children: []
+      endpoint:
+        type: REST
+      name: classifier
+      type: MODEL
+    name: noeng
+    replicas: 1

operator/api/v1alpha2/seldondeployment_types.go

@@ -43,6 +43,7 @@ const (
 	ANNOTATION_JAVA_OPTS       = "seldon.io/engine-java-opts"
 	ANNOTATION_SEPARATE_ENGINE = "seldon.io/engine-separate-pod"
 	ANNOTATION_HEADLESS_SVC    = "seldon.io/headless-svc"
+	ANNOTATION_NO_ENGINE       = "seldon.io/no-engine"
 	ANNOTATION_CUSTOM_SVC_NAME = "seldon.io/svc-name"
 )
 

operator/api/v1alpha2/seldondeployment_webhook.go

@@ -398,11 +398,26 @@ func checkTraffic(mlDep *SeldonDeployment, fldPath *field.Path, allErrs field.Er
 	return allErrs
 }
 
+func sizeOfGraph(p *PredictiveUnit) int {
+	count := 0
+	for _, child := range p.Children {
+		count = count + sizeOfGraph(&child)
+	}
+	return count + 1
+}
+
 func (r *SeldonDeployment) validateSeldonDeployment() error {
 	var allErrs field.ErrorList
 
 	predictorNames := make(map[string]bool)
 	for i, p := range r.Spec.Predictors {
+
+		_, noEngine := p.Annotations[ANNOTATION_NO_ENGINE]
+		if noEngine && sizeOfGraph(p.Graph) > 1 {
+			fldPath := field.NewPath("spec").Child("predictors").Index(i)
+			allErrs = append(allErrs, field.Invalid(fldPath, p.Name, "Running without engine only valid for single element graphs"))
+		}
+
 		if _, present := predictorNames[p.Name]; present {
 			fldPath := field.NewPath("spec").Child("predictors").Index(i)
 			allErrs = append(allErrs, field.Invalid(fldPath, p.Name, "Duplicate predictor name"))

operator/controllers/ambassador.go

@@ -252,10 +252,27 @@ func getAmbassadorConfigs(mlDep *machinelearningv1alpha2.SeldonDeployment, p *ma
 			return "", err
 		}
 
-		if GetEnv("AMBASSADOR_SINGLE_NAMESPACE", "false") == "true" {
-			return YAML_SEP + cRestGlobal + YAML_SEP + cGrpcGlobal + YAML_SEP + cRestNamespaced + YAML_SEP + cGrpcNamespaced, nil
+		// Return the appropriate set of config based on whether http and/or grpc is active
+		if engine_http_port > 0 && engine_grpc_port > 0 {
+			if GetEnv("AMBASSADOR_SINGLE_NAMESPACE", "false") == "true" {
+				return YAML_SEP + cRestGlobal + YAML_SEP + cGrpcGlobal + YAML_SEP + cRestNamespaced + YAML_SEP + cGrpcNamespaced, nil
+			} else {
+				return YAML_SEP + cRestGlobal + YAML_SEP + cGrpcGlobal, nil
+			}
+		} else if engine_http_port > 0 {
+			if GetEnv("AMBASSADOR_SINGLE_NAMESPACE", "false") == "true" {
+				return YAML_SEP + cRestGlobal + YAML_SEP + cRestNamespaced, nil
+			} else {
+				return YAML_SEP + cRestGlobal, nil
+			}
+		} else if engine_grpc_port > 0 {
+			if GetEnv("AMBASSADOR_SINGLE_NAMESPACE", "false") == "true" {
+				return YAML_SEP + cGrpcGlobal + YAML_SEP + cGrpcNamespaced, nil
+			} else {
+				return YAML_SEP + cGrpcGlobal, nil
+			}
 		} else {
-			return YAML_SEP + cRestGlobal + YAML_SEP + cGrpcGlobal, nil
+			return "", nil
 		}
 
 	}

operator/controllers/noengine_test.go

@@ -0,0 +1,87 @@
+package controllers
+
+import (
+	"context"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	machinelearningv1alpha2 "github.com/seldonio/seldon-core/operator/api/v1alpha2"
+	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"time"
+)
+
+var _ = Describe("Create a Seldon Deployment without engine", func() {
+	const timeout = time.Second * 30
+	const interval = time.Second * 1
+	By("Creating a resource")
+	It("should create a resource with defaults", func() {
+		Expect(k8sClient).NotTo(BeNil())
+		var modelType = machinelearningv1alpha2.MODEL
+		key := types.NamespacedName{
+			Name:      "dep2",
+			Namespace: "default",
+		}
+		instance := &machinelearningv1alpha2.SeldonDeployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      key.Name,
+				Namespace: key.Namespace,
+			},
+			Spec: machinelearningv1alpha2.SeldonDeploymentSpec{
+				Name: "mydep2",
+				Predictors: []machinelearningv1alpha2.PredictorSpec{
+					{
+						Annotations: map[string]string{
+							"seldon.io/no-engine": "true",
+						},
+						Name: "p1",
+						ComponentSpecs: []*machinelearningv1alpha2.SeldonPodSpec{
+							{
+								Spec: v1.PodSpec{
+									Containers: []v1.Container{
+										{
+											Image: "seldonio/mock_classifier:1.0",
+											Name:  "classifier",
+										},
+									},
+								},
+							},
+						},
+						Graph: &machinelearningv1alpha2.PredictiveUnit{
+							Name: "classifier",
+							Type: &modelType,
+						},
+					},
+				},
+			},
+		}
+
+		// Run Defaulter
+		instance.Default()
+
+		Expect(k8sClient.Create(context.Background(), instance)).Should(Succeed())
+		//time.Sleep(time.Second * 5)
+
+		fetched := &machinelearningv1alpha2.SeldonDeployment{}
+		Eventually(func() error {
+			err := k8sClient.Get(context.Background(), key, fetched)
+			return err
+		}, timeout, interval).Should(BeNil())
+		Expect(fetched.Spec.Name).Should(Equal("mydep2"))
+
+		depKey := types.NamespacedName{
+			Name:      machinelearningv1alpha2.GetDeploymentName(instance, instance.Spec.Predictors[0], instance.Spec.Predictors[0].ComponentSpecs[0]),
+			Namespace: "default",
+		}
+		depFetched := &appsv1.Deployment{}
+		Eventually(func() error {
+			err := k8sClient.Get(context.Background(), depKey, depFetched)
+			return err
+		}, timeout, interval).Should(BeNil())
+		Expect(len(depFetched.Spec.Template.Spec.Containers)).Should(Equal(1))
+
+		Expect(k8sClient.Delete(context.Background(), instance)).Should(Succeed())
+	})
+
+})