Upgrade Go version/Knative/Spark Operator/Kubernetes Client/Docker Compose for dev environment.

.github/actions/run-cluster-init/action.yaml

@@ -31,15 +31,15 @@ runs:
   using: composite
   steps:
     - name: "Setup local k8s cluster"
-      uses: AbsaOSS/k3d-action@v1.5.0
+      uses: AbsaOSS/k3d-action@v2.0.0
       with:
         cluster-name: ${{ inputs.cluster_name }}
-        use-default-registry: true
         args: >-
           --servers 1
           --agents 3
-          --port 80:80@loadbalancer
-          --k3s-server-arg "--no-deploy=traefik,metrics-server"
+          --port 80:80@agent:*
+          --registry-create ${{inputs.local_registry}}
+          --k3s-arg "--no-deploy=traefik,metrics-server@server:*"
 
     - name: Publish images to local registry
       env:

.github/workflows/turing.yaml

@@ -211,10 +211,10 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v2
 
-      - name: Set up Go 1.14
+      - name: Set up Go 1.16
         uses: actions/setup-go@v2
         with:
-          go-version: "1.14"
+          go-version: "1.16"
 
       - name: Cache Dependencies
         uses: actions/cache@v2
@@ -254,10 +254,10 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v2
 
-      - name: Set up Go 1.14
+      - name: Set up Go 1.16
         uses: actions/setup-go@v2
         with:
-          go-version: "1.14"
+          go-version: "1.16"
 
       - name: Cache Dependencies
         uses: actions/cache@v2
@@ -289,10 +289,10 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v2
 
-      - name: Set up Go 1.14
+      - name: Set up Go 1.16
         uses: actions/setup-go@v2
         with:
-          go-version: "1.14"
+          go-version: "1.16"
 
       - name: Cache Dependencies
         uses: actions/cache@v2
@@ -349,10 +349,10 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v2
 
-      - name: Set up Go 1.14
+      - name: Set up Go 1.16
         uses: actions/setup-go@v2
         with:
-          go-version: "1.14"
+          go-version: "1.16"
 
       - name: Download Turing API Docker tar archive
         uses: actions/download-artifact@v2
@@ -374,10 +374,10 @@ jobs:
         with:
           name: cluster-init.${{ env.CLUSTER_INIT_VERSION }}.tar
 
-      - name: Set up Go 1.14
+      - name: Set up Go 1.16
         uses: actions/setup-go@v2
         with:
-          go-version: "1.14"
+          go-version: "1.16"
 
       - name: Run action cluster-init
         uses: ./.github/actions/run-cluster-init
@@ -433,7 +433,9 @@ jobs:
           VAULT_TOKEN: root
         run: |
           echo "::group::Wait for Vault to become available"
-          timeout --foreground 45 bash -c \
+          # TODO: remove
+          curl -v -H "X-Vault-Token: ${{ env.VAULT_TOKEN }}" http://vault.127.0.0.1.nip.io/v1/sys/mounts
+          timeout --foreground 100 bash -c \
             'until curl -s --fail -H "X-Vault-Token: ${{ env.VAULT_TOKEN }}" http://vault.127.0.0.1.nip.io/v1/sys/mounts; do sleep 2; done'
           echo "::endgroup::"
 
@@ -446,6 +448,8 @@ jobs:
           }
           EOF
 
+          echo credentials.json
+
           curl -v \
             --header "X-Vault-Token: ${{ env.VAULT_TOKEN }}" \
             --request POST \
@@ -481,7 +485,7 @@ jobs:
             deployment/turing-mlp,
             deployment/turing-merlin,
             deployment/turing
-          timeout: 300 #seconds
+          timeout: 600 # seconds
           max-restarts: -1
 
       - name: Setup MLP project
@@ -522,25 +526,33 @@ jobs:
           KUBECONFIG_USE_LOCAL: true
         run: go test -v -parallel=2 ./e2e/... -tags=e2e -run TestEndToEnd
 
-      - if: (steps.wait-for-deployment.outcome == 'failure' || steps.run-e2e-test.outcome == 'failure') && always()
+      - if: (steps.wait-for-deployment.outcome == 'failure'  || steps.run-e2e-test.outcome == 'failure') && always()
         name: "Debug Deployment Failure"
         run: |
+          echo "::group::describe deployment/turing-mlp"
+          kubectl describe deployment/turing-mlp
+          echo "::endgroup::"
+
           echo "::group::describe deployment/turing-merlin"
           kubectl describe deployment/turing-merlin
           echo "::endgroup::"
-          
+
           echo "::group::describe deployment/turing"
           kubectl describe deployment/turing
           echo "::endgroup::"
-          
+
           echo "::group::secret/turing-api-config"
           kubectl get secret/turing-api-config -o jsonpath='{.data.config\.yaml}' | base64 --decode
           echo "::endgroup::"
-          
+
           echo "::group::logs deployment/turing-merlin"
           kubectl logs deployment/turing-merlin
           echo "::endgroup::"
-
+
+          echo "::group::logs deployment/turing-mlp"
+          kubectl logs deployment/turing-mlp
+          echo "::endgroup::"
+
           echo "::group::logs deployment/turing"
           kubectl logs deployment/turing
           echo "::endgroup::"

api/Dockerfile

@@ -1,5 +1,5 @@
 # Build turing-api binary
-FROM golang:1.14-alpine as api-builder
+FROM golang:1.16-alpine as api-builder
 ARG API_BIN_NAME=turing-api
 
 ENV GO111MODULE=on \

api/config-dev.yaml

@@ -24,7 +24,7 @@ BatchEnsemblingConfig:
       BuildContextURI: git://github.com/gojek/turing.git#refs/heads/main
       DockerfileFilePath: engines/pyfunc-ensembler-job/app.Dockerfile
       Image: gcr.io/kaniko-project/executor
-      ImageVersion: v1.6.0
+      ImageVersion: v1.8.0
       ResourceRequestsLimits:
         Requests:
           CPU: "1"

api/e2e/test/01_create_router_test.go

@@ -1,3 +1,4 @@
+//go:build e2e
 // +build e2e
 
 package e2e

api/e2e/test/02_update_router_invalid_test.go

@@ -1,3 +1,4 @@
+//go:build e2e
 // +build e2e
 
 package e2e

api/e2e/test/04_undeploy_router_test.go

@@ -1,3 +1,4 @@
+//go:build e2e
 // +build e2e
 
 package e2e

api/e2e/test/05_deploy_invalid_config_test.go

@@ -1,3 +1,4 @@
+//go:build e2e
 // +build e2e
 
 package e2e

api/e2e/test/06_deploy_valid_config_test.go

@@ -1,3 +1,4 @@
+//go:build e2e
 // +build e2e
 
 package e2e

api/e2e/test/07_delete_router_test.go

@@ -1,3 +1,4 @@
+//go:build e2e
 // +build e2e
 
 package e2e

api/e2e/test/08_deploy_router_with_traffic_rules_test.go

@@ -1,3 +1,4 @@
+//go:build e2e
 // +build e2e
 
 package e2e

api/e2e/test/helpers_api_test.go

@@ -1,3 +1,4 @@
+//go:build e2e
 // +build e2e
 
 package e2e

api/e2e/test/helpers_cluster_test.go

@@ -1,8 +1,10 @@
+//go:build e2e
 // +build e2e
 
 package e2e
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
 
@@ -11,7 +13,7 @@ import (
 
 	"github.com/gojek/mlp/api/pkg/vault"
 	"github.com/pkg/errors"
-	networkingv1alpha3 "istio.io/client-go/pkg/clientset/versioned/typed/networking/v1alpha3"
+	networkingv1beta1 "istio.io/client-go/pkg/clientset/versioned/typed/networking/v1beta1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
@@ -27,7 +29,7 @@ type TestClusterClients struct {
 	KnServingClient       knservingclient.ServingV1Interface
 	K8sCoreClient         corev1.CoreV1Interface
 	K8sAppsClient         appsv1.AppsV1Interface
-	IstioNetworkingClient networkingv1alpha3.NetworkingV1alpha3Interface
+	IstioNetworkingClient networkingv1beta1.NetworkingV1beta1Interface
 }
 
 // newClusterClients is a creator for the TestClusterClients
@@ -79,7 +81,7 @@ func newClusterClients(cfg *testConfig) (*TestClusterClients, error) {
 	if err != nil {
 		return nil, err
 	}
-	istioClientset, err := networkingv1alpha3.NewForConfig(clusterCfg)
+	istioClientset, err := networkingv1beta1.NewForConfig(clusterCfg)
 	if err != nil {
 		return nil, err
 	}
@@ -98,7 +100,7 @@ func isConfigMapExists(
 ) bool {
 	_, err := clusterClients.K8sCoreClient.
 		ConfigMaps(projectName).
-		Get(name, metav1.GetOptions{})
+		Get(context.Background(), name, metav1.GetOptions{})
 	return err == nil
 }
 
@@ -108,7 +110,7 @@ func isPersistentVolumeClaimExists(clusterClients *TestClusterClients,
 ) bool {
 	_, err := clusterClients.K8sCoreClient.
 		PersistentVolumeClaims(projectName).
-		Get(name, metav1.GetOptions{})
+		Get(context.Background(), name, metav1.GetOptions{})
 	return err == nil
 }
 
@@ -119,12 +121,15 @@ func isDeploymentExists(
 ) bool {
 	_, err := clusterClients.K8sAppsClient.
 		Deployments(projectName).
-		Get(name, metav1.GetOptions{})
+		Get(context.Background(), name, metav1.GetOptions{})
 	return err == nil
 }
 
 func getRouterDownstream(clusterClients *TestClusterClients, projectName string, routerName string) (string, error) {
-	vs, err := clusterClients.IstioNetworkingClient.VirtualServices(projectName).Get(routerName, metav1.GetOptions{})
+	vs, err := clusterClients.
+		IstioNetworkingClient.
+		VirtualServices(projectName).
+		Get(context.Background(), routerName, metav1.GetOptions{})
 	if err != nil {
 		return "", err
 	}
@@ -189,13 +194,13 @@ func deleteServices(
 	serviceNames []string,
 ) {
 	gracePeriodSeconds := int64(deleteTimeoutSeconds)
-	delOptions := &metav1.DeleteOptions{
+	delOptions := metav1.DeleteOptions{
 		GracePeriodSeconds: &gracePeriodSeconds,
 	}
 
 	services := clusterClients.K8sCoreClient.Services(projectName)
 	for _, name := range serviceNames {
-		err := services.Delete(name, delOptions)
+		err := services.Delete(context.Background(), name, delOptions)
 		if err != nil && !k8serrors.IsNotFound(err) {
 			fmt.Printf("Error deleting service %s: %v\n", name, err)
 		}
@@ -208,13 +213,13 @@ func deleteDeployments(
 	deploymentNames []string,
 ) {
 	gracePeriodSeconds := int64(deleteTimeoutSeconds)
-	delOptions := &metav1.DeleteOptions{
+	delOptions := metav1.DeleteOptions{
 		GracePeriodSeconds: &gracePeriodSeconds,
 	}
 
 	deployments := clusterClients.K8sAppsClient.Deployments(projectName)
 	for _, name := range deploymentNames {
-		err := deployments.Delete(name, delOptions)
+		err := deployments.Delete(context.Background(), name, delOptions)
 		if err != nil && !k8serrors.IsNotFound(err) {
 			fmt.Printf("Error deleting deployment %s: %v\n", name, err)
 		}
@@ -227,13 +232,13 @@ func deleteKnServices(
 	knServiceNames []string,
 ) {
 	gracePeriodSeconds := int64(deleteTimeoutSeconds)
-	delOptions := &metav1.DeleteOptions{
+	delOptions := metav1.DeleteOptions{
 		GracePeriodSeconds: &gracePeriodSeconds,
 	}
 
 	services := clusterClients.KnServingClient.Services(projectName)
 	for _, name := range knServiceNames {
-		err := services.Delete(name, delOptions)
+		err := services.Delete(context.Background(), name, delOptions)
 		if err != nil && !k8serrors.IsNotFound(err) {
 			fmt.Printf("Error deleting Knative service %s: %v\n", name, err)
 		}
@@ -246,13 +251,13 @@ func deleteSecrets(
 	secretNames []string,
 ) {
 	gracePeriodSeconds := int64(deleteTimeoutSeconds)
-	delOptions := &metav1.DeleteOptions{
+	delOptions := metav1.DeleteOptions{
 		GracePeriodSeconds: &gracePeriodSeconds,
 	}
 
 	secrets := clusterClients.K8sCoreClient.Secrets(projectName)
 	for _, name := range secretNames {
-		err := secrets.Delete(name, delOptions)
+		err := secrets.Delete(context.Background(), name, delOptions)
 		if err != nil && !k8serrors.IsNotFound(err) {
 			fmt.Printf("Error deleting secret %s: %v\n", name, err)
 		}
@@ -265,13 +270,13 @@ func deleteConfigmaps(
 	cfgMapNames []string,
 ) {
 	gracePeriodSeconds := int64(deleteTimeoutSeconds)
-	delOptions := &metav1.DeleteOptions{
+	delOptions := metav1.DeleteOptions{
 		GracePeriodSeconds: &gracePeriodSeconds,
 	}
 
 	configMaps := clusterClients.K8sCoreClient.ConfigMaps(projectName)
 	for _, name := range cfgMapNames {
-		err := configMaps.Delete(name, delOptions)
+		err := configMaps.Delete(context.Background(), name, delOptions)
 		if err != nil && !k8serrors.IsNotFound(err) {
 			fmt.Printf("Error deleting configmap %s: %v\n", name, err)
 		}
@@ -284,13 +289,13 @@ func deletePVCs(
 	pvcs []string,
 ) {
 	gracePeriodSeconds := int64(deleteTimeoutSeconds)
-	delOptions := &metav1.DeleteOptions{
+	delOptions := metav1.DeleteOptions{
 		GracePeriodSeconds: &gracePeriodSeconds,
 	}
 
 	clusterPVCs := clusterClients.K8sCoreClient.PersistentVolumeClaims(projectName)
 	for _, name := range pvcs {
-		err := clusterPVCs.Delete(name, delOptions)
+		err := clusterPVCs.Delete(context.Background(), name, delOptions)
 		if err != nil && !k8serrors.IsNotFound(err) {
 			fmt.Printf("Error deleting PVC %s: %v\n", name, err)
 		}
@@ -303,13 +308,13 @@ func deleteIstioVirtualServices(
 	svcs []string,
 ) {
 	gracePeriodSeconds := int64(deleteTimeoutSeconds)
-	delOptions := &metav1.DeleteOptions{
+	delOptions := metav1.DeleteOptions{
 		GracePeriodSeconds: &gracePeriodSeconds,
 	}
 
 	vservices := clusterClients.IstioNetworkingClient.VirtualServices(projectName)
 	for _, name := range svcs {
-		err := vservices.Delete(name, delOptions)
+		err := vservices.Delete(context.Background(), name, delOptions)
 		if err != nil && !k8serrors.IsNotFound(err) {
 			fmt.Printf("Error deleting Istio Virtual Service %s: %v\n", name, err)
 		}

api/e2e/test/helpers_test.go

@@ -1,3 +1,4 @@
+//go:build e2e
 // +build e2e
 
 package e2e