Merge remote-tracking branch 'upstream/main' into enable-wi-kaniko

* upstream/main: Refactor deployed model labels (#346) [Pyfunc] Add ability to push metrics to Prometheus push gateway (#311)
caraml-dev · Feb 21, 2023 · 427fba3 · 427fba3
2 parents ff92a6c + 14d23c9
commit 427fba3
Show file tree

Hide file tree

Showing 34 changed files with 821 additions and 293 deletions.
diff --git a/Makefile b/Makefile
@@ -56,7 +56,7 @@ lint-ui:
 .PHONY: lint-api
 lint-api:
 	@echo "> Analyzing API source code..."
-	@cd ${API_PATH} && golangci-lint run 
+	@cd ${API_PATH} && golangci-lint run
 
 # ============================================================
 # Testing recipes

diff --git a/api/api/router_test.go b/api/api/router_test.go
@@ -16,7 +16,7 @@ package api
 
 import (
 	"bytes"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -440,7 +440,7 @@ func Test_prometheusMiddleware_post(t *testing.T) {
 	router := mux.NewRouter()
 
 	router.HandleFunc("/foo", func(rw http.ResponseWriter, r *http.Request) {
-		body, err := ioutil.ReadAll(r.Body)
+		body, err := io.ReadAll(r.Body)
 		assert.Nil(t, err)
 
 		_, err = rw.Write(body)
@@ -476,7 +476,7 @@ func Test_prometheusMiddleware_post_500(t *testing.T) {
 	router := mux.NewRouter()
 
 	router.HandleFunc("/foo-500", func(rw http.ResponseWriter, r *http.Request) {
-		body, err := ioutil.ReadAll(r.Body)
+		body, err := io.ReadAll(r.Body)
 		assert.Nil(t, err)
 
 		rw.WriteHeader(500)

diff --git a/api/batch/resource_test.go b/api/batch/resource_test.go
@@ -15,7 +15,6 @@
 package batch
 
 import (
-	"fmt"
 	"testing"
 
 	"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/apis/sparkoperator.k8s.io/v1beta2"
@@ -52,24 +51,26 @@ var (
 	modelID   = models.ID(2)
 	versionID = models.ID(3)
 
-	labelTeamName         = "gojek.com/team"
-	labelStreamName       = "gojek.com/stream"
 	labelAppName          = "gojek.com/app"
-	labelOrchestratorName = "gojek.com/orchestrator"
+	labelComponentName    = "gojek.com/component"
 	labelEnvironment      = "gojek.com/environment"
-	labelUsersPrefix      = "gojek.com/%s"
+	labelOrchestratorName = "gojek.com/orchestrator"
+	labelStreamName       = "gojek.com/stream"
+	labelTeamName         = "gojek.com/team"
 
 	defaultLabels = map[string]string{
+		labelModelID:         modelID.String(),
+		labelModelVersionID:  versionID.String(),
+		labelPredictionJobID: jobID.String(),
+
+		labelAppName:          modelName,
+		labelComponentName:    models.ComponentBatchJob,
+		labelEnvironment:      environementName,
 		labelOrchestratorName: "merlin",
-		labelModelID:          modelID.String(),
-		labelModelVersionID:   versionID.String(),
-		labelPredictionJobID:  jobID.String(),
+		labelStreamName:       streamName,
+		labelTeamName:         teamName,
 
-		labelTeamName:                           teamName,
-		labelStreamName:                         streamName,
-		labelAppName:                            modelName,
-		labelEnvironment:                        environementName,
-		fmt.Sprintf(labelUsersPrefix, "my-key"): "my-value",
+		"my-key": "my-value",
 	}
 
 	driverCore       int32 = 1
@@ -118,6 +119,9 @@ var (
 )
 
 func TestCreateSparkApplicationResource(t *testing.T) {
+	models.InitKubernetesLabeller("gojek.com/") //nolint:errcheck
+	defer models.InitKubernetesLabeller("")     //nolint:errcheck
+
 	tests := []struct {
 		name           string
 		arg            *models.PredictionJob
@@ -131,10 +135,11 @@ func TestCreateSparkApplicationResource(t *testing.T) {
 				Name: jobName,
 				ID:   jobID,
 				Metadata: models.Metadata{
-					Team:        teamName,
-					Stream:      streamName,
 					App:         modelName,
+					Component:   models.ComponentBatchJob,
 					Environment: environementName,
+					Stream:      streamName,
+					Team:        teamName,
 					Labels:      userLabels,
 				},
 				VersionModelID: modelID,
@@ -210,10 +215,11 @@ func TestCreateSparkApplicationResource(t *testing.T) {
 				Name: jobName,
 				ID:   jobID,
 				Metadata: models.Metadata{
-					Team:        teamName,
-					Stream:      streamName,
 					App:         modelName,
+					Component:   models.ComponentBatchJob,
 					Environment: environementName,
+					Stream:      streamName,
+					Team:        teamName,
 					Labels:      userLabels,
 				},
 				VersionModelID: modelID,
@@ -289,10 +295,11 @@ func TestCreateSparkApplicationResource(t *testing.T) {
 				Name: jobName,
 				ID:   jobID,
 				Metadata: models.Metadata{
-					Team:        teamName,
-					Stream:      streamName,
 					App:         modelName,
+					Component:   models.ComponentBatchJob,
 					Environment: environementName,
+					Stream:      streamName,
+					Team:        teamName,
 					Labels:      userLabels,
 				},
 				VersionModelID: modelID,
@@ -368,10 +375,11 @@ func TestCreateSparkApplicationResource(t *testing.T) {
 				Name: jobName,
 				ID:   jobID,
 				Metadata: models.Metadata{
-					Team:        teamName,
-					Stream:      streamName,
 					App:         modelName,
+					Component:   models.ComponentBatchJob,
 					Environment: environementName,
+					Stream:      streamName,
+					Team:        teamName,
 					Labels:      userLabels,
 				},
 				VersionModelID: modelID,
@@ -447,10 +455,11 @@ func TestCreateSparkApplicationResource(t *testing.T) {
 				Name: jobName,
 				ID:   jobID,
 				Metadata: models.Metadata{
-					Team:        teamName,
-					Stream:      streamName,
 					App:         modelName,
+					Component:   models.ComponentBatchJob,
 					Environment: environementName,
+					Stream:      streamName,
+					Team:        teamName,
 					Labels:      userLabels,
 				},
 				VersionModelID: modelID,
@@ -538,10 +547,11 @@ func TestCreateSparkApplicationResource(t *testing.T) {
 				Name: jobName,
 				ID:   jobID,
 				Metadata: models.Metadata{
-					Team:        teamName,
-					Stream:      streamName,
 					App:         modelName,
+					Component:   models.ComponentBatchJob,
 					Environment: environementName,
+					Stream:      streamName,
+					Team:        teamName,
 					Labels:      userLabels,
 				},
 				VersionModelID: modelID,
@@ -568,10 +578,11 @@ func TestCreateSparkApplicationResource(t *testing.T) {
 				Name: jobName,
 				ID:   jobID,
 				Metadata: models.Metadata{
-					Team:        teamName,
-					Stream:      streamName,
 					App:         modelName,
+					Component:   models.ComponentBatchJob,
 					Environment: environementName,
+					Stream:      streamName,
+					Team:        teamName,
 					Labels:      userLabels,
 				},
 				VersionModelID: modelID,
@@ -598,10 +609,11 @@ func TestCreateSparkApplicationResource(t *testing.T) {
 				Name: jobName,
 				ID:   jobID,
 				Metadata: models.Metadata{
-					Team:        teamName,
-					Stream:      streamName,
 					App:         modelName,
+					Component:   models.ComponentBatchJob,
 					Environment: environementName,
+					Stream:      streamName,
+					Team:        teamName,
 					Labels:      userLabels,
 				},
 				VersionModelID: modelID,

diff --git a/api/cluster/controller.go b/api/cluster/controller.go
@@ -112,7 +112,8 @@ func newController(kfservingClient kservev1beta1client.ServingV1beta1Interface,
 	batchV1Client batchv1client.BatchV1Interface,
 	deploymentConfig config.DeploymentConfig,
 	containerFetcher ContainerFetcher,
-	templater *resource.InferenceServiceTemplater) (Controller, error) {
+	templater *resource.InferenceServiceTemplater,
+) (Controller, error) {
 	return &controller{
 		servingClient:              kfservingClient,
 		clusterClient:              coreV1Client,
@@ -197,6 +198,7 @@ func (k *controller) Deploy(ctx context.Context, modelService *models.Service) (
 		Namespace:   s.Namespace,
 		ServiceName: s.Status.URL.Host,
 		URL:         inferenceURL,
+		Metadata:    modelService.Metadata,
 	}, nil
 }