Refactor deployed model labels (#346)

<!--  Thanks for sending a pull request!  Here are some tips for you:

1. Run unit tests and ensure that they are passing
2. If your change introduces any API changes, make sure to update the
e2e tests
3. Make sure documentation is updated for your PR!

-->

**What this PR does / why we need it**:
<!-- Explain here the context and why you're making the change. What is
the problem you're trying to solve. --->

The labels implementation of Merlin resources (model services, batch
jobs) is having two major limitations:
1. Hard coded to use "gojek.com/" as prefix so user can change it
2. Only propagate project's labels even though version labels are
available via
[new_model_version()](https://github.com/gojek/merlin/blob/main/python/sdk/merlin/fluent.py#L157)
function.

These drawbacks limit the user to have more flexibility on using label
for their tracking.

This PR make following changes:
1. Allow user to specify label prefix
2. Merge version labels with project labels. Version labels have more
priority.
3. Introduce "component" labels to tag model-version, model-endpoint,
batch-job, and image-builder resources.

**Which issue(s) this PR fixes**:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
-->

Fixes #

**Does this PR introduce a user-facing change?**:
<!--
If no, just write "NONE" in the release-note block below.
If yes, a release note is required. Enter your extended release note in
the block below.
If the PR requires additional action from users switching to the new
release, include the string "action required".

For more information about release notes, see kubernetes' guide here:
http://git.k8s.io/community/contributors/guide/release-notes.md
-->

```release-note

```

**Checklist**

- [x] Added unit test, integration, and/or e2e tests
- [ ] Tested locally
- [ ] Updated documentation
- [ ] Update Swagger spec if the PR introduce API changes
- [ ] Regenerated Golang and Python client if the PR introduce API
changes

---------

Co-authored-by: yadavsunny05 <yadav.sunny05@gmail.com>

Makefile

@@ -56,7 +56,7 @@ lint-ui:
 .PHONY: lint-api
 lint-api:
 	@echo "> Analyzing API source code..."
-	@cd ${API_PATH} && golangci-lint run 
+	@cd ${API_PATH} && golangci-lint run
 
 # ============================================================
 # Testing recipes

api/api/router_test.go

@@ -16,7 +16,7 @@ package api
 
 import (
 	"bytes"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -440,7 +440,7 @@ func Test_prometheusMiddleware_post(t *testing.T) {
 	router := mux.NewRouter()
 
 	router.HandleFunc("/foo", func(rw http.ResponseWriter, r *http.Request) {
-		body, err := ioutil.ReadAll(r.Body)
+		body, err := io.ReadAll(r.Body)
 		assert.Nil(t, err)
 
 		_, err = rw.Write(body)
@@ -476,7 +476,7 @@ func Test_prometheusMiddleware_post_500(t *testing.T) {
 	router := mux.NewRouter()
 
 	router.HandleFunc("/foo-500", func(rw http.ResponseWriter, r *http.Request) {
-		body, err := ioutil.ReadAll(r.Body)
+		body, err := io.ReadAll(r.Body)
 		assert.Nil(t, err)
 
 		rw.WriteHeader(500)

api/batch/resource_test.go

@@ -15,7 +15,6 @@
 package batch
 
 import (
-	"fmt"
 	"testing"
 
 	"github.com/GoogleCloudPlatform/spark-on-k8s-operator/pkg/apis/sparkoperator.k8s.io/v1beta2"
@@ -52,24 +51,26 @@ var (
 	modelID   = models.ID(2)
 	versionID = models.ID(3)
 
-	labelTeamName         = "gojek.com/team"
-	labelStreamName       = "gojek.com/stream"
 	labelAppName          = "gojek.com/app"
-	labelOrchestratorName = "gojek.com/orchestrator"
+	labelComponentName    = "gojek.com/component"
 	labelEnvironment      = "gojek.com/environment"
-	labelUsersPrefix      = "gojek.com/%s"
+	labelOrchestratorName = "gojek.com/orchestrator"
+	labelStreamName       = "gojek.com/stream"
+	labelTeamName         = "gojek.com/team"
 
 	defaultLabels = map[string]string{
+		labelModelID:         modelID.String(),
+		labelModelVersionID:  versionID.String(),
+		labelPredictionJobID: jobID.String(),
+
+		labelAppName:          modelName,
+		labelComponentName:    models.ComponentBatchJob,
+		labelEnvironment:      environementName,
 		labelOrchestratorName: "merlin",
-		labelModelID:          modelID.String(),
-		labelModelVersionID:   versionID.String(),
-		labelPredictionJobID:  jobID.String(),
+		labelStreamName:       streamName,
+		labelTeamName:         teamName,
 
-		labelTeamName:                           teamName,
-		labelStreamName:                         streamName,
-		labelAppName:                            modelName,
-		labelEnvironment:                        environementName,
-		fmt.Sprintf(labelUsersPrefix, "my-key"): "my-value",
+		"my-key": "my-value",
 	}
 
 	driverCore       int32 = 1
@@ -118,6 +119,9 @@ var (
 )
 
 func TestCreateSparkApplicationResource(t *testing.T) {
+	models.InitKubernetesLabeller("gojek.com/") //nolint:errcheck
+	defer models.InitKubernetesLabeller("")     //nolint:errcheck
+
 	tests := []struct {
 		name           string
 		arg            *models.PredictionJob
@@ -131,10 +135,11 @@ func TestCreateSparkApplicationResource(t *testing.T) {
 				Name: jobName,
 				ID:   jobID,
 				Metadata: models.Metadata{
-					Team:        teamName,
-					Stream:      streamName,
 					App:         modelName,
+					Component:   models.ComponentBatchJob,
 					Environment: environementName,
+					Stream:      streamName,
+					Team:        teamName,
 					Labels:      userLabels,
 				},
 				VersionModelID: modelID,
@@ -210,10 +215,11 @@ func TestCreateSparkApplicationResource(t *testing.T) {
 				Name: jobName,
 				ID:   jobID,
 				Metadata: models.Metadata{
-					Team:        teamName,
-					Stream:      streamName,
 					App:         modelName,
+					Component:   models.ComponentBatchJob,
 					Environment: environementName,
+					Stream:      streamName,
+					Team:        teamName,
 					Labels:      userLabels,
 				},
 				VersionModelID: modelID,
@@ -289,10 +295,11 @@ func TestCreateSparkApplicationResource(t *testing.T) {
 				Name: jobName,
 				ID:   jobID,
 				Metadata: models.Metadata{
-					Team:        teamName,
-					Stream:      streamName,
 					App:         modelName,
+					Component:   models.ComponentBatchJob,
 					Environment: environementName,
+					Stream:      streamName,
+					Team:        teamName,
 					Labels:      userLabels,
 				},
 				VersionModelID: modelID,
@@ -368,10 +375,11 @@ func TestCreateSparkApplicationResource(t *testing.T) {
 				Name: jobName,
 				ID:   jobID,
 				Metadata: models.Metadata{
-					Team:        teamName,
-					Stream:      streamName,
 					App:         modelName,
+					Component:   models.ComponentBatchJob,
 					Environment: environementName,
+					Stream:      streamName,
+					Team:        teamName,
 					Labels:      userLabels,
 				},
 				VersionModelID: modelID,
@@ -447,10 +455,11 @@ func TestCreateSparkApplicationResource(t *testing.T) {
 				Name: jobName,
 				ID:   jobID,
 				Metadata: models.Metadata{
-					Team:        teamName,
-					Stream:      streamName,
 					App:         modelName,
+					Component:   models.ComponentBatchJob,
 					Environment: environementName,
+					Stream:      streamName,
+					Team:        teamName,
 					Labels:      userLabels,
 				},
 				VersionModelID: modelID,
@@ -538,10 +547,11 @@ func TestCreateSparkApplicationResource(t *testing.T) {
 				Name: jobName,
 				ID:   jobID,
 				Metadata: models.Metadata{
-					Team:        teamName,
-					Stream:      streamName,
 					App:         modelName,
+					Component:   models.ComponentBatchJob,
 					Environment: environementName,
+					Stream:      streamName,
+					Team:        teamName,
 					Labels:      userLabels,
 				},
 				VersionModelID: modelID,
@@ -568,10 +578,11 @@ func TestCreateSparkApplicationResource(t *testing.T) {
 				Name: jobName,
 				ID:   jobID,
 				Metadata: models.Metadata{
-					Team:        teamName,
-					Stream:      streamName,
 					App:         modelName,
+					Component:   models.ComponentBatchJob,
 					Environment: environementName,
+					Stream:      streamName,
+					Team:        teamName,
 					Labels:      userLabels,
 				},
 				VersionModelID: modelID,
@@ -598,10 +609,11 @@ func TestCreateSparkApplicationResource(t *testing.T) {
 				Name: jobName,
 				ID:   jobID,
 				Metadata: models.Metadata{
-					Team:        teamName,
-					Stream:      streamName,
 					App:         modelName,
+					Component:   models.ComponentBatchJob,
 					Environment: environementName,
+					Stream:      streamName,
+					Team:        teamName,
 					Labels:      userLabels,
 				},
 				VersionModelID: modelID,

api/cluster/controller.go

@@ -112,7 +112,8 @@ func newController(kfservingClient kservev1beta1client.ServingV1beta1Interface,
 	batchV1Client batchv1client.BatchV1Interface,
 	deploymentConfig config.DeploymentConfig,
 	containerFetcher ContainerFetcher,
-	templater *resource.InferenceServiceTemplater) (Controller, error) {
+	templater *resource.InferenceServiceTemplater,
+) (Controller, error) {
 	return &controller{
 		servingClient:              kfservingClient,
 		clusterClient:              coreV1Client,
@@ -197,6 +198,7 @@ func (k *controller) Deploy(ctx context.Context, modelService *models.Service) (
 		Namespace:   s.Namespace,
 		ServiceName: s.Status.URL.Host,
 		URL:         inferenceURL,
+		Metadata:    modelService.Metadata,
 	}, nil
 }