Add integration test for skaffold dev

I added an integration test to make sure a Job is deleted and redeployed
upon changes when running via skaffold dev. The test sets up by creating
a file foo. It runs skaffold dev and make sure the Job is created. It
then changes foo so that skaffold redeploys, and makes sure the UID of
the new Job is different from the UID of the old job.

examples/helm-deployment/skaffold.yaml

@@ -4,7 +4,7 @@ build:
   tagPolicy:
     sha256: {}
   artifacts:
-  - imageName: gcr.io/k8s-skaffold/skaffold-helm
+  - imageName: gcr.io/priya-wadhwa/skaffold-helm
 deploy:
   helm:
     releases:
@@ -13,7 +13,7 @@ deploy:
       #wait: true
       #valuesFilePath: helm-skaffold-values.yaml
       values:
-        image: gcr.io/k8s-skaffold/skaffold-helm
+        image: gcr.io/priya-wadhwa/skaffold-helm
       #overrides builds an override values.yaml file to run with the helm deploy
       #overrides:
       # some:

examples/test-dev-job/Dockerfile

@@ -0,0 +1,8 @@
+FROM golang:1.10.1-alpine3.7 as builder
+COPY main.go .
+RUN go build -o /app main.go
+
+FROM alpine:3.7  
+CMD ["./app"]
+COPY --from=builder /app .
+COPY foo /foo

examples/test-dev-job/k8s-job.yaml

@@ -0,0 +1,11 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: test-dev-job
+spec:
+  template:
+    spec:
+      containers:
+      - name: test-dev-job
+        image: gcr.io/k8s-skaffold/test-dev-job
+      restartPolicy: OnFailure

examples/test-dev-job/main.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"fmt"
+	"time"
+)
+
+func main() {
+	for {
+		fmt.Println("Hello world!!")
+		time.Sleep(time.Second * 1)
+	}
+}

examples/test-dev-job/skaffold.yaml

@@ -0,0 +1,14 @@
+apiVersion: skaffold/v1alpha2
+kind: Config
+build:
+  artifacts:
+  - imageName: gcr.io/k8s-skaffold/test-dev-job
+deploy:
+  kubectl:
+    manifests:
+      - k8s-*
+profiles:
+  - name: gcb
+    build:
+      googleCloudBuild:
+        projectId: k8s-skaffold

integration/run_test.go

@@ -30,8 +30,10 @@ import (
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
 	"github.com/sirupsen/logrus"
 	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
 	"k8s.io/api/core/v1"
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 )
 
@@ -185,19 +187,116 @@ func TestRun(t *testing.T) {
 			}
 
 			// Cleanup
-			args = []string{"delete", "--namespace", ns.Name}
-			if testCase.filename != "" {
-				args = append(args, "-f", testCase.filename)
-			}
-			cmd = exec.Command("skaffold", args...)
+			cleanupTest(t, ns, testCase.dir, testCase.filename)
+		})
+	}
+}
+
+func TestDev(t *testing.T) {
+	type testDevCase struct {
+		description   string
+		dir           string
+		args          []string
+		setup         func(t *testing.T) func(t *testing.T)
+		jobs          []string
+		jobValidation func(t *testing.T, ns *v1.Namespace, j *batchv1.Job)
+	}
+
+	testCases := []testDevCase{
+		{
+			description: "delete and redeploy job",
+			dir:         "../examples/test-dev-job",
+			args:        []string{"dev"},
+			setup: func(t *testing.T) func(t *testing.T) {
+				// create foo
+				cmd := exec.Command("touch", "../examples/test-dev-job/foo")
+				if output, err := util.RunCmdOut(cmd); err != nil {
+					t.Fatalf("creating foo: %s %v", output, err)
+				}
+				return func(t *testing.T) {
+					// delete foo
+					cmd := exec.Command("rm", "../examples/test-dev-job/foo")
+					if output, err := util.RunCmdOut(cmd); err != nil {
+						t.Fatalf("creating foo: %s %v", output, err)
+					}
+				}
+			},
+			jobs: []string{
+				"test-dev-job",
+			},
+			jobValidation: func(t *testing.T, ns *v1.Namespace, j *batchv1.Job) {
+				originalUID := j.GetUID()
+				// Make a change to foo so that dev is forced to delete the job and redeploy
+				cmd := exec.Command("sh", "-c", "echo bar > ../examples/test-dev-job/foo")
+				if output, err := util.RunCmdOut(cmd); err != nil {
+					t.Fatalf("creating bar: %s %v", output, err)
+				}
+				// Make sure the UID of the old Job and the UID of the new Job is different
+				err := wait.PollImmediate(time.Millisecond*500, 10*time.Minute, func() (bool, error) {
+					newJob, err := client.BatchV1().Jobs(ns.Name).Get(j.Name, meta_v1.GetOptions{})
+					if err != nil {
+						return false, nil
+					}
+					return originalUID != newJob.GetUID(), nil
+				})
+				if err != nil {
+					t.Fatalf("original UID and new UID are the same, redeploy failed")
+				}
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.description, func(t *testing.T) {
+			ns, deleteNs := setupNamespace(t)
+			defer deleteNs()
+
+			cleanupTC := testCase.setup(t)
+			defer cleanupTC(t)
+
+			args := []string{}
+			args = append(args, testCase.args...)
+			args = append(args, "--namespace", ns.Name)
+
+			cmd := exec.Command("skaffold", args...)
 			cmd.Dir = testCase.dir
-			if output, err := util.RunCmdOut(cmd); err != nil {
-				t.Fatalf("skaffold delete: %s %v", output, err)
+			go func() {
+				if output, err := util.RunCmdOut(cmd); err != nil {
+					t.Fatalf("skaffold: %s %v", output, err)
+				}
+			}()
+
+			for _, j := range testCase.jobs {
+				if err := kubernetesutil.WaitForJobToStabilize(client, ns.Name, j, 10*time.Minute); err != nil {
+					t.Fatalf("Timed out waiting for job to stabilize")
+				}
+				if testCase.jobValidation != nil {
+					job, err := client.BatchV1().Jobs(ns.Name).Get(j, meta_v1.GetOptions{})
+					if err != nil {
+						t.Fatalf("Could not find job: %s %s", ns.Name, j)
+					}
+					testCase.jobValidation(t, ns, job)
+				}
 			}
+
+			// Cleanup
+			cleanupTest(t, ns, testCase.dir, "")
 		})
 	}
 }
 
+func cleanupTest(t *testing.T, ns *v1.Namespace, dir, filename string) {
+	args := []string{"delete", "--namespace", ns.Name}
+	if filename != "" {
+		args = append(args, "-f", filename)
+	}
+	cmd := exec.Command("skaffold", args...)
+	cmd.Dir = dir
+	if output, err := util.RunCmdOut(cmd); err != nil {
+		t.Fatalf("skaffold delete: %s %v", output, err)
+	}
+}
+
 func setupNamespace(t *testing.T) (*v1.Namespace, func()) {
 	ns, err := client.CoreV1().Namespaces().Create(&v1.Namespace{
 		ObjectMeta: meta_v1.ObjectMeta{

pkg/skaffold/kubernetes/wait.go

@@ -123,3 +123,17 @@ func WaitForDeploymentToStabilize(c kubernetes.Interface, ns, name string, timeo
 	})
 	return err
 }
+
+// WaitForJobToStabilize waits till the Job has at least one active pod
+func WaitForJobToStabilize(c kubernetes.Interface, ns, name string, timeout time.Duration) error {
+	return wait.PollImmediate(time.Millisecond*500, timeout, func() (bool, error) {
+		job, err := c.BatchV1().Jobs(ns).Get(name, meta_v1.GetOptions{})
+		if err != nil {
+			return false, nil
+		}
+		if job.Status.Active > 0 {
+			return true, nil
+		}
+		return false, nil
+	})
+}