From 3674813c5aeb2e5c01736b93bd87808cb611b15c Mon Sep 17 00:00:00 2001
From: Nick Kubala <nkubala@google.com>
Date: Tue, 18 Sep 2018 14:57:23 -0700
Subject: [PATCH 1/9] Add test runner

 This PR adds a test phase to the runner. If any tests are specified,
 they will always be executed after the build phase but before the deploy
 phase, and if any tests fail the deploy phase will be skipped. In the
 dev loop, only newly built images will be tested; any artifacts that
 were not modified by new changes will be skipped.

 A 'test' section is added to the config, where users can add all of
 their test configuration. Each test is specified at the artifact level,
 matched up by image names specified in the build artifacts.

 The test executor is designed to be pluggable, similar to the builders
 and deployers. The main exception is that any number of testers can be
 executed in a single run; a failure in any of them counts as a failure
 of the entire test run. This PR implements running structure tests,
 but adds the infrastructure to easily add more test types in the future.
---
 deploy/skaffold/Dockerfile                    |  5 +-
 .../profile_structure_test.yaml               |  6 ++
 examples/getting-started/skaffold.yaml        |  9 +++
 examples/getting-started/structure_test.yaml  |  6 ++
 pkg/skaffold/runner/runner.go                 | 39 +++++++++-
 pkg/skaffold/runner/timings.go                | 18 ++++-
 pkg/skaffold/schema/latest/config.go          |  9 +++
 pkg/skaffold/schema/profiles.go               |  1 +
 pkg/skaffold/schema/v1alpha3/config.go        | 16 +++-
 pkg/skaffold/test/structure/structure.go      | 39 ++++++++++
 pkg/skaffold/test/structure/types.go          | 27 +++++++
 pkg/skaffold/test/test.go                     | 78 +++++++++++++++++++
 pkg/skaffold/test/types.go                    | 63 +++++++++++++++
 13 files changed, 308 insertions(+), 8 deletions(-)
 create mode 100644 examples/getting-started/profile_structure_test.yaml
 create mode 100644 examples/getting-started/structure_test.yaml
 create mode 100644 pkg/skaffold/test/structure/structure.go
 create mode 100644 pkg/skaffold/test/structure/types.go
 create mode 100644 pkg/skaffold/test/test.go
 create mode 100644 pkg/skaffold/test/types.go

diff --git a/deploy/skaffold/Dockerfile b/deploy/skaffold/Dockerfile
index f9921e50fdb..16370c60ded 100644
--- a/deploy/skaffold/Dockerfile
+++ b/deploy/skaffold/Dockerfile
@@ -47,6 +47,10 @@ RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8
 RUN apt-get update \
   && apt-get install -y bazel
 
+RUN curl -LO https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 \
+  && chmod +x container-structure-test-linux-amd64 \
+  && mv container-structure-test-linux-amd64 /usr/local/bin/container-structure-test
+
 ENV PATH /usr/local/go/bin:/go/bin:/google-cloud-sdk/bin:$PATH
 
 FROM runtime_deps as builder
@@ -94,4 +98,3 @@ CMD ["make", "integration"]
 FROM runtime_deps as distribution
 
 COPY --from=integration /usr/bin/skaffold /usr/bin/skaffold
-
diff --git a/examples/getting-started/profile_structure_test.yaml b/examples/getting-started/profile_structure_test.yaml
new file mode 100644
index 00000000000..6966d26930a
--- /dev/null
+++ b/examples/getting-started/profile_structure_test.yaml
@@ -0,0 +1,6 @@
+schemaVersion: 2.0.0
+
+fileExistenceTests:
+  - name: 'no go binary'
+    path: '/usr/bin/go'
+    shouldExist: false
diff --git a/examples/getting-started/skaffold.yaml b/examples/getting-started/skaffold.yaml
index fb4d4adbc4c..48361be1a3e 100644
--- a/examples/getting-started/skaffold.yaml
+++ b/examples/getting-started/skaffold.yaml
@@ -3,6 +3,10 @@ kind: Config
 build:
   artifacts:
   - imageName: gcr.io/k8s-skaffold/skaffold-example
+test:
+  - imageName: gcr.io/k8s-skaffold/skaffold-example
+    structureTests:
+      - structure_test.yaml
 deploy:
   kubectl:
     manifests:
@@ -12,3 +16,8 @@ profiles:
     build:
       googleCloudBuild:
         projectId: k8s-skaffold
+  - name: test
+    test:
+      - imageName: gcr.io/k8s-skaffold/skaffold-example
+        structureTests:
+          - profile_structure_test.yaml
diff --git a/examples/getting-started/structure_test.yaml b/examples/getting-started/structure_test.yaml
new file mode 100644
index 00000000000..6b1223eae90
--- /dev/null
+++ b/examples/getting-started/structure_test.yaml
@@ -0,0 +1,6 @@
+schemaVersion: 2.0.0
+
+fileExistenceTests:
+  - name: 'no local go binary'
+    path: /usr/local/bin/go'
+    shouldExist: false
diff --git a/pkg/skaffold/runner/runner.go b/pkg/skaffold/runner/runner.go
index cf50520f9eb..5814567a157 100644
--- a/pkg/skaffold/runner/runner.go
+++ b/pkg/skaffold/runner/runner.go
@@ -38,6 +38,7 @@ import (
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
 	kubectx "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/context"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/test"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/watch"
 
 	"github.com/pkg/errors"
@@ -51,6 +52,7 @@ var ErrorConfigurationChanged = errors.New("configuration changed")
 type SkaffoldRunner struct {
 	build.Builder
 	deploy.Deployer
+	test.Tester
 	tag.Tagger
 
 	opts         *config.SkaffoldOptions
@@ -76,19 +78,25 @@ func NewForConfig(opts *config.SkaffoldOptions, cfg *latest.SkaffoldConfig) (*Sk
 		return nil, errors.Wrap(err, "parsing skaffold build config")
 	}
 
+	tester, err := getTester(&cfg.Test)
+	if err != nil {
+		return nil, errors.Wrap(err, "parsing skaffold test config")
+	}
+
 	deployer, err := getDeployer(&cfg.Deploy, kubeContext, opts.Namespace)
 	if err != nil {
 		return nil, errors.Wrap(err, "parsing skaffold deploy config")
 	}
 
 	deployer = deploy.WithLabels(deployer, opts, builder, deployer, tagger)
-	builder, deployer = WithTimings(builder, deployer)
+	builder, tester, deployer = WithTimings(builder, tester, deployer)
 	if opts.Notification {
 		deployer = WithNotification(deployer)
 	}
 
 	return &SkaffoldRunner{
 		Builder:      builder,
+		Tester:       tester,
 		Deployer:     deployer,
 		Tagger:       tagger,
 		opts:         opts,
@@ -115,6 +123,10 @@ func getBuilder(cfg *latest.BuildConfig, kubeContext string) (build.Builder, err
 	}
 }
 
+func getTester(cfg *[]latest.TestCase) (test.Tester, error) {
+	return test.NewTester(cfg)
+}
+
 func getDeployer(cfg *latest.DeployConfig, kubeContext string, namespace string) (deploy.Deployer, error) {
 	deployers := []deploy.Deployer{}
 
@@ -171,13 +183,17 @@ func getTagger(t latest.TagPolicy, customTag string) (tag.Tagger, error) {
 	}
 }
 
-// Run builds artifacts and then deploys them.
+// Run builds artifacts, runs tests on built artifacts, and then deploys them.
 func (r *SkaffoldRunner) Run(ctx context.Context, out io.Writer, artifacts []*latest.Artifact) error {
 	bRes, err := r.Build(ctx, out, r.Tagger, artifacts)
 	if err != nil {
 		return errors.Wrap(err, "build step")
 	}
 
+	if err = r.Test(out, bRes); err != nil {
+		return errors.Wrap(err, "test step")
+	}
+
 	_, err = r.Deploy(ctx, out, bRes)
 	if err != nil {
 		return errors.Wrap(err, "deploy step")
@@ -241,12 +257,20 @@ func (r *SkaffoldRunner) Dev(ctx context.Context, out io.Writer, artifacts []*la
 			}
 
 			r.updateBuiltImages(imageList, bRes)
+			if err := r.Test(out, bRes); err != nil {
+				logrus.Warnln("Skipping Deploy due to failed tests:", err)
+				return nil
+			}
 
 			if _, err = r.Deploy(ctx, out, r.builds); err != nil {
 				logrus.Warnln("Skipping Deploy due to error:", err)
 				return nil
 			}
 		case changed.needsRedeploy:
+			if err := r.Test(out, r.builds); err != nil {
+				logrus.Warnln("Skipping Deploy due to failed tests:", err)
+				return nil
+			}
 			if _, err := r.Deploy(ctx, out, r.builds); err != nil {
 				logrus.Warnln("Skipping Deploy due to error:", err)
 				return nil
@@ -275,6 +299,14 @@ func (r *SkaffoldRunner) Dev(ctx context.Context, out io.Writer, artifacts []*la
 		}
 	}
 
+	// Watch test configuration
+	if err := watcher.Register(
+		func() ([]string, error) { return r.TestDependencies(), nil },
+		func(watch.Events) { changed.needsRedeploy = true },
+	); err != nil {
+		return nil, errors.Wrap(err, "watching test files")
+	}
+
 	// Watch deployment configuration
 	if err := watcher.Register(
 		func() ([]string, error) { return r.Dependencies() },
@@ -298,6 +330,9 @@ func (r *SkaffoldRunner) Dev(ctx context.Context, out io.Writer, artifacts []*la
 	}
 
 	r.updateBuiltImages(imageList, bRes)
+	if err := r.Test(out, bRes); err != nil {
+		return nil, errors.Wrap(err, "exiting dev mode because the first test run failed")
+	}
 
 	_, err = r.Deploy(ctx, out, r.builds)
 	if err != nil {
diff --git a/pkg/skaffold/runner/timings.go b/pkg/skaffold/runner/timings.go
index 53713cbc982..0bf13b0b000 100644
--- a/pkg/skaffold/runner/timings.go
+++ b/pkg/skaffold/runner/timings.go
@@ -26,20 +26,23 @@ import (
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/color"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/deploy"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/test"
 )
 
 // WithTimings creates a deployer that logs the duration of each phase.
-func WithTimings(b build.Builder, d deploy.Deployer) (build.Builder, deploy.Deployer) {
+func WithTimings(b build.Builder, t test.Tester, d deploy.Deployer) (build.Builder, test.Tester, deploy.Deployer) {
 	w := withTimings{
 		Builder:  b,
+		Tester:   t,
 		Deployer: d,
 	}
 
-	return w, w
+	return w, w, w
 }
 
 type withTimings struct {
 	build.Builder
+	test.Tester
 	deploy.Deployer
 }
 
@@ -54,6 +57,17 @@ func (w withTimings) Build(ctx context.Context, out io.Writer, tagger tag.Tagger
 	return bRes, err
 }
 
+func (w withTimings) Test(out io.Writer, builds []build.Artifact) error {
+	start := time.Now()
+	color.Default.Fprintln(out, "Starting test...")
+
+	err := w.Tester.Test(out, builds)
+	if err == nil {
+		color.Default.Fprintln(out, "Test complete in", time.Since(start))
+	}
+	return err
+}
+
 func (w withTimings) Deploy(ctx context.Context, out io.Writer, builds []build.Artifact) ([]deploy.Artifact, error) {
 	start := time.Now()
 	color.Default.Fprintln(out, "Starting deploy...")
diff --git a/pkg/skaffold/schema/latest/config.go b/pkg/skaffold/schema/latest/config.go
index b1bd8b1011d..2c650e754d1 100644
--- a/pkg/skaffold/schema/latest/config.go
+++ b/pkg/skaffold/schema/latest/config.go
@@ -34,6 +34,7 @@ type SkaffoldConfig struct {
 	Kind       string `yaml:"kind"`
 
 	Build    BuildConfig  `yaml:"build,omitempty"`
+	Test     []TestCase   `yaml:"test,omitempty"`
 	Deploy   DeployConfig `yaml:"deploy,omitempty"`
 	Profiles []Profile    `yaml:"profiles,omitempty"`
 }
@@ -116,6 +117,13 @@ type KanikoBuild struct {
 	Timeout        string             `yaml:"timeout,omitempty"`
 }
 
+// TestCase is a struct containing all the specified test
+// configuration for an image
+type TestCase struct {
+	ImageName      string   `yaml:"image"`
+	StructureTests []string `yaml:"structureTests,omitempty"`
+}
+
 // DeployConfig contains all the configuration needed by the deploy steps
 type DeployConfig struct {
 	DeployType `yaml:",inline"`
@@ -212,6 +220,7 @@ type Artifact struct {
 type Profile struct {
 	Name   string       `yaml:"name"`
 	Build  BuildConfig  `yaml:"build,omitempty"`
+	Test   []TestCase   `yaml:"test,omitempty"`
 	Deploy DeployConfig `yaml:"deploy,omitempty"`
 }
 
diff --git a/pkg/skaffold/schema/profiles.go b/pkg/skaffold/schema/profiles.go
index 58823355c42..fd191393151 100644
--- a/pkg/skaffold/schema/profiles.go
+++ b/pkg/skaffold/schema/profiles.go
@@ -55,6 +55,7 @@ func applyProfile(config *latest.SkaffoldConfig, profile latest.Profile) {
 		Kind:       config.Kind,
 		Build:      overlayProfileField(config.Build, profile.Build).(latest.BuildConfig),
 		Deploy:     overlayProfileField(config.Deploy, profile.Deploy).(latest.DeployConfig),
+		Test:       overlayProfileField(config.Test, profile.Test).([]latest.TestCase),
 	}
 }
 
diff --git a/pkg/skaffold/schema/v1alpha3/config.go b/pkg/skaffold/schema/v1alpha3/config.go
index cb35735b426..b02fb52f42a 100644
--- a/pkg/skaffold/schema/v1alpha3/config.go
+++ b/pkg/skaffold/schema/v1alpha3/config.go
@@ -34,6 +34,7 @@ type SkaffoldConfig struct {
 	Kind       string `yaml:"kind"`
 
 	Build    BuildConfig  `yaml:"build,omitempty"`
+	Test     []TestCase   `yaml:"test,omitempty"`
 	Deploy   DeployConfig `yaml:"deploy,omitempty"`
 	Profiles []Profile    `yaml:"profiles,omitempty"`
 }
@@ -116,6 +117,13 @@ type KanikoBuild struct {
 	Timeout        string             `yaml:"timeout,omitempty"`
 }
 
+// TestCase is a struct containing all the specified test
+// configuration for an image.
+type TestCase struct {
+	ImageName      string   `yaml:"imageName"`
+	StructureTests []string `yaml:"structureTests,omitempty"`
+}
+
 // DeployConfig contains all the configuration needed by the deploy steps
 type DeployConfig struct {
 	DeployType `yaml:",inline"`
@@ -201,9 +209,10 @@ type HelmConventionConfig struct {
 // Artifact represents items that need to be built, along with the context in which
 // they should be built.
 type Artifact struct {
-	ImageName    string `yaml:"imageName"`
-	Workspace    string `yaml:"workspace,omitempty"`
-	ArtifactType `yaml:",inline"`
+	ImageName          string `yaml:"imageName"`
+	Workspace          string `yaml:"workspace,omitempty"`
+	ArtifactType       `yaml:",inline"`
+	StructureTestFiles []string `yaml:"structureTestFiles"`
 }
 
 // Profile is additional configuration that overrides default
@@ -211,6 +220,7 @@ type Artifact struct {
 type Profile struct {
 	Name   string       `yaml:"name"`
 	Build  BuildConfig  `yaml:"build,omitempty"`
+	Test   []TestCase   `yaml:"test,omitempty"`
 	Deploy DeployConfig `yaml:"deploy,omitempty"`
 }
 
diff --git a/pkg/skaffold/test/structure/structure.go b/pkg/skaffold/test/structure/structure.go
new file mode 100644
index 00000000000..6179dfdf5a8
--- /dev/null
+++ b/pkg/skaffold/test/structure/structure.go
@@ -0,0 +1,39 @@
+/*
+Copyright 2018 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package structure
+
+import (
+	"os/exec"
+
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
+
+	"github.com/sirupsen/logrus"
+)
+
+// Test is the entrypoint for running structure tests
+func (tr *TestRunner) Test(image string) error {
+	logrus.Infof("running structure tests for files %v", tr.testFiles)
+	args := []string{"test", "--image", image}
+	for _, f := range tr.testFiles {
+		args = append(args, "--config", f)
+	}
+	args = append(args, tr.testFiles...)
+	cmd := exec.Command("container-structure-test", args...)
+
+	_, err := util.RunCmdOut(cmd)
+	return err
+}
diff --git a/pkg/skaffold/test/structure/types.go b/pkg/skaffold/test/structure/types.go
new file mode 100644
index 00000000000..af1fb804432
--- /dev/null
+++ b/pkg/skaffold/test/structure/types.go
@@ -0,0 +1,27 @@
+/*
+Copyright 2018 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package structure
+
+type TestRunner struct {
+	testFiles []string
+}
+
+func NewStructureTestRunner(files []string) (*TestRunner, error) {
+	return &TestRunner{
+		testFiles: files,
+	}, nil
+}
diff --git a/pkg/skaffold/test/test.go b/pkg/skaffold/test/test.go
new file mode 100644
index 00000000000..1fa7b1a768c
--- /dev/null
+++ b/pkg/skaffold/test/test.go
@@ -0,0 +1,78 @@
+/*
+Copyright 2018 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package test
+
+import (
+	"io"
+
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v1alpha3"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/test/structure"
+
+	"github.com/pkg/errors"
+)
+
+// Test is the top level testing execution call.
+// it should return back the list of artifacts that passed the tests
+// in some form, so they can be deployed. It should also be responsible for
+// logging which artifacts were not deployed because their tests failed.
+func (t FullTester) Test(out io.Writer, bRes []build.Artifact) error {
+	t.resolveArtifactImageTags(bRes)
+	for _, aTester := range t.ArtifactTesters {
+		if err := aTester.RunTests(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// NewTester parses the provided test cases from the Skaffold config,
+// and returns a Tester instance with all the necessary test runners
+// to run all specified tests.
+func NewTester(testCases *[]v1alpha3.TestCase) (Tester, error) {
+	testers := []*ArtifactTester{}
+	deps := []string{}
+	for _, testCase := range *testCases {
+		testRunner := &ArtifactTester{
+			ImageName: testCase.ImageName,
+		}
+		if testCase.StructureTests != nil {
+			stRunner, err := structure.NewStructureTestRunner(testCase.StructureTests)
+			if err != nil {
+				return FullTester{}, errors.Wrap(err, "retrieving structure test runner")
+			}
+			testRunner.TestRunners = append(testRunner.TestRunners, stRunner)
+			deps = append(deps, testCase.StructureTests...)
+		}
+		testers = append(testers, testRunner)
+	}
+	return FullTester{
+		ArtifactTesters: testers,
+		Dependencies:    deps,
+	}, nil
+}
+
+// replace original test artifact images with tagged build artifact images
+func (t *FullTester) resolveArtifactImageTags(bRes []build.Artifact) {
+	for _, aTest := range t.ArtifactTesters {
+		for _, res := range bRes {
+			if aTest.ImageName == res.ImageName {
+				aTest.ImageName = res.Tag
+			}
+		}
+	}
+}
diff --git a/pkg/skaffold/test/types.go b/pkg/skaffold/test/types.go
new file mode 100644
index 00000000000..6cb9f3a0aae
--- /dev/null
+++ b/pkg/skaffold/test/types.go
@@ -0,0 +1,63 @@
+/*
+Copyright 2018 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package test
+
+import (
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
+	"io"
+)
+
+// Tester is the top level test executor in Skaffold.
+// A tester is really a collection of artifact-specific testers,
+// each of which contains one or more TestRunners which implements
+// a single test run.
+type Tester interface {
+	Test(io.Writer, []build.Artifact) error
+
+	TestDependencies() []string
+}
+
+type FullTester struct {
+	ArtifactTesters []*ArtifactTester
+	Dependencies    []string
+}
+
+func (t FullTester) TestDependencies() []string {
+	return t.Dependencies
+}
+
+// ArtifactTester is an artifact-specific test holder, which contains
+// tests runners to run all specified tests on an individual artifact.
+type ArtifactTester struct {
+	ImageName   string
+	TestRunners []TestRunner
+}
+
+func (a *ArtifactTester) RunTests() error {
+	for _, t := range a.TestRunners {
+		if err := t.Test(a.ImageName); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// TestRunner is the lowest-level test executor in Skaffold, responsible for
+// running a single test on a single artifact image and returning its result.
+type TestRunner interface {
+	Test(image string) error
+}

From d888488e5b0318338570e1f933d0d5cdf0b37fdb Mon Sep 17 00:00:00 2001
From: Nick Kubala <nkubala@google.com>
Date: Fri, 21 Sep 2018 11:57:31 -0700
Subject: [PATCH 2/9] cleanup and organization

---
 pkg/skaffold/test/test.go  | 42 +++++++++++++++++++++++++-------------
 pkg/skaffold/test/types.go | 24 +++++++++-------------
 2 files changed, 38 insertions(+), 28 deletions(-)

diff --git a/pkg/skaffold/test/test.go b/pkg/skaffold/test/test.go
index 1fa7b1a768c..bd37056106d 100644
--- a/pkg/skaffold/test/test.go
+++ b/pkg/skaffold/test/test.go
@@ -26,20 +26,6 @@ import (
 	"github.com/pkg/errors"
 )
 
-// Test is the top level testing execution call.
-// it should return back the list of artifacts that passed the tests
-// in some form, so they can be deployed. It should also be responsible for
-// logging which artifacts were not deployed because their tests failed.
-func (t FullTester) Test(out io.Writer, bRes []build.Artifact) error {
-	t.resolveArtifactImageTags(bRes)
-	for _, aTester := range t.ArtifactTesters {
-		if err := aTester.RunTests(); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 // NewTester parses the provided test cases from the Skaffold config,
 // and returns a Tester instance with all the necessary test runners
 // to run all specified tests.
@@ -66,6 +52,34 @@ func NewTester(testCases *[]v1alpha3.TestCase) (Tester, error) {
 	}, nil
 }
 
+// TestDependencies returns the watch dependencies to the runner.
+func (t FullTester) TestDependencies() []string {
+	return t.Dependencies
+}
+
+// Test is the top level testing execution call. It serves as the
+// entrypoint to all individual tests.
+func (t FullTester) Test(out io.Writer, bRes []build.Artifact) error {
+	t.resolveArtifactImageTags(bRes)
+	for _, aTester := range t.ArtifactTesters {
+		if err := aTester.RunTests(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// RunTests serves as the entrypoint to each group of
+// artifact-specific tests.
+func (a *ArtifactTester) RunTests() error {
+	for _, t := range a.TestRunners {
+		if err := t.Test(a.ImageName); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // replace original test artifact images with tagged build artifact images
 func (t *FullTester) resolveArtifactImageTags(bRes []build.Artifact) {
 	for _, aTest := range t.ArtifactTesters {
diff --git a/pkg/skaffold/test/types.go b/pkg/skaffold/test/types.go
index 6cb9f3a0aae..a4938e22f88 100644
--- a/pkg/skaffold/test/types.go
+++ b/pkg/skaffold/test/types.go
@@ -17,8 +17,9 @@ limitations under the License.
 package test
 
 import (
-	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
 	"io"
+
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
 )
 
 // Tester is the top level test executor in Skaffold.
@@ -31,15 +32,18 @@ type Tester interface {
 	TestDependencies() []string
 }
 
+// FullTester serves as a holder for the individual artifact-specific
+// testers. It exists so that the Tester interface can mimic the Builder/Deployer
+// interface, so it can be called in a similar fashion from the Runner, while
+// the FullTester actually handles the work.
+
+// FullTester should always be the ONLY implementation of the Tester interface;
+// newly added testing implementations should implement the TestRunner interface.
 type FullTester struct {
 	ArtifactTesters []*ArtifactTester
 	Dependencies    []string
 }
 
-func (t FullTester) TestDependencies() []string {
-	return t.Dependencies
-}
-
 // ArtifactTester is an artifact-specific test holder, which contains
 // tests runners to run all specified tests on an individual artifact.
 type ArtifactTester struct {
@@ -47,17 +51,9 @@ type ArtifactTester struct {
 	TestRunners []TestRunner
 }
 
-func (a *ArtifactTester) RunTests() error {
-	for _, t := range a.TestRunners {
-		if err := t.Test(a.ImageName); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 // TestRunner is the lowest-level test executor in Skaffold, responsible for
 // running a single test on a single artifact image and returning its result.
+// Any new test type should implement this interface.
 type TestRunner interface {
 	Test(image string) error
 }

From b0a83d6ae8ff000e7cc33fb27afe02bb693981f6 Mon Sep 17 00:00:00 2001
From: Nick Kubala <nkubala@google.com>
Date: Fri, 21 Sep 2018 12:43:21 -0700
Subject: [PATCH 3/9] add tests

---
 pkg/skaffold/runner/runner_test.go | 69 +++++++++++++++++++++++++++++-
 1 file changed, 68 insertions(+), 1 deletion(-)

diff --git a/pkg/skaffold/runner/runner_test.go b/pkg/skaffold/runner/runner_test.go
index 9276893851c..02c6fa6a617 100644
--- a/pkg/skaffold/runner/runner_test.go
+++ b/pkg/skaffold/runner/runner_test.go
@@ -31,6 +31,7 @@ import (
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/deploy"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/test"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/watch"
 	"github.com/GoogleContainerTools/skaffold/testutil"
 	clientgo "k8s.io/client-go/kubernetes"
@@ -65,6 +66,23 @@ func (t *TestBuilder) Build(ctx context.Context, w io.Writer, tagger tag.Tagger,
 	return builds, nil
 }
 
+type TestTester struct {
+	errors []error
+}
+
+func (t *TestTester) Test(out io.Writer, builds []build.Artifact) error {
+	if len(t.errors) > 0 {
+		err := t.errors[0]
+		t.errors = t.errors[1:]
+		return err
+	}
+	return nil
+}
+
+func (t *TestTester) TestDependencies() []string {
+	return []string{}
+}
+
 type TestDeployer struct {
 	deployed []build.Artifact
 	errors   []error
@@ -140,6 +158,7 @@ func TestNewForConfig(t *testing.T) {
 		config           *latest.SkaffoldConfig
 		shouldErr        bool
 		expectedBuilder  build.Builder
+		expectedTester   test.Tester
 		expectedDeployer deploy.Deployer
 	}{
 		{
@@ -158,6 +177,7 @@ func TestNewForConfig(t *testing.T) {
 				},
 			},
 			expectedBuilder:  &local.Builder{},
+			expectedTester:   &test.FullTester{},
 			expectedDeployer: &deploy.KubectlDeployer{},
 		},
 		{
@@ -184,6 +204,7 @@ func TestNewForConfig(t *testing.T) {
 			},
 			shouldErr:        true,
 			expectedBuilder:  &local.Builder{},
+			expectedTester:   &test.FullTester{},
 			expectedDeployer: &deploy.KubectlDeployer{},
 		},
 		{
@@ -197,6 +218,7 @@ func TestNewForConfig(t *testing.T) {
 				}},
 			shouldErr:        true,
 			expectedBuilder:  &local.Builder{},
+			expectedTester:   &test.FullTester{},
 			expectedDeployer: &deploy.KubectlDeployer{},
 		},
 		{
@@ -218,7 +240,7 @@ func TestNewForConfig(t *testing.T) {
 
 			testutil.CheckError(t, test.shouldErr, err)
 			if cfg != nil {
-				b, d := WithTimings(test.expectedBuilder, test.expectedDeployer)
+				b, _, d := WithTimings(test.expectedBuilder, test.expectedTester, test.expectedDeployer)
 
 				testutil.CheckErrorAndTypeEquality(t, test.shouldErr, err, b, cfg.Builder)
 				testutil.CheckErrorAndTypeEquality(t, test.shouldErr, err, d, cfg.Deployer)
@@ -232,6 +254,7 @@ func TestRun(t *testing.T) {
 		description string
 		config      *latest.SkaffoldConfig
 		builder     build.Builder
+		tester      test.Tester
 		deployer    deploy.Deployer
 		shouldErr   bool
 	}{
@@ -239,6 +262,7 @@ func TestRun(t *testing.T) {
 			description: "run no error",
 			config:      &latest.SkaffoldConfig{},
 			builder:     &TestBuilder{},
+			tester:      &TestTester{},
 			deployer:    &TestDeployer{},
 		},
 		{
@@ -247,6 +271,7 @@ func TestRun(t *testing.T) {
 			builder: &TestBuilder{
 				errors: []error{fmt.Errorf("")},
 			},
+			tester:    &TestTester{},
 			shouldErr: true,
 		},
 		{
@@ -261,17 +286,42 @@ func TestRun(t *testing.T) {
 				},
 			},
 			builder: &TestBuilder{},
+			tester:  &TestTester{},
 			deployer: &TestDeployer{
 				errors: []error{fmt.Errorf("")},
 			},
 			shouldErr: true,
 		},
+		{
+			description: "run test error",
+			config: &v1alpha3.SkaffoldConfig{
+				Build: v1alpha3.BuildConfig{
+					Artifacts: []*v1alpha3.Artifact{
+						{
+							ImageName: "test",
+						},
+					},
+				},
+				Test: []v1alpha3.TestCase{
+					{
+						ImageName:      "test",
+						StructureTests: []string{"fake_file.yaml"},
+					},
+				},
+			},
+			builder: &TestBuilder{},
+			tester: &TestTester{
+				errors: []error{fmt.Errorf("")},
+			},
+			shouldErr: true,
+		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.description, func(t *testing.T) {
 			runner := &SkaffoldRunner{
 				Builder:  test.builder,
+				Tester:   test.tester,
 				Deployer: test.deployer,
 				Tagger:   &tag.ChecksumTagger{},
 				opts:     &config.SkaffoldOptions{},
@@ -290,6 +340,7 @@ func TestDev(t *testing.T) {
 	var tests = []struct {
 		description    string
 		builder        build.Builder
+		tester         test.Tester
 		deployer       deploy.Deployer
 		watcherFactory watch.Factory
 		shouldErr      bool
@@ -306,23 +357,35 @@ func TestDev(t *testing.T) {
 		{
 			description: "fails to deploy the first time",
 			builder:     &TestBuilder{},
+			tester:      &TestTester{},
 			deployer: &TestDeployer{
 				errors: []error{fmt.Errorf("")},
 			},
 			watcherFactory: NewWatcherFactory(nil, nil),
 			shouldErr:      true,
 		},
+		{
+			description: "fails to deploy due to failed tests",
+			builder:     &TestBuilder{},
+			tester: &TestTester{
+				errors: []error{fmt.Errorf("")},
+			},
+			watcherFactory: NewWatcherFactory(nil),
+			shouldErr:      true,
+		},
 		{
 			description: "ignore subsequent build errors",
 			builder: &TestBuilder{
 				errors: []error{nil, fmt.Errorf("")},
 			},
+			tester:         &TestTester{},
 			deployer:       &TestDeployer{},
 			watcherFactory: NewWatcherFactory(nil, nil, nil),
 		},
 		{
 			description: "ignore subsequent deploy errors",
 			builder:     &TestBuilder{},
+			tester:      &TestTester{},
 			deployer: &TestDeployer{
 				errors: []error{nil, fmt.Errorf("")},
 			},
@@ -331,6 +394,7 @@ func TestDev(t *testing.T) {
 		{
 			description:    "fail to watch files",
 			builder:        &TestBuilder{},
+			tester:         &TestTester{},
 			deployer:       &TestDeployer{},
 			watcherFactory: NewWatcherFactory(fmt.Errorf(""), nil),
 			shouldErr:      true,
@@ -341,6 +405,7 @@ func TestDev(t *testing.T) {
 		t.Run(test.description, func(t *testing.T) {
 			runner := &SkaffoldRunner{
 				Builder:      test.builder,
+				Tester:       test.tester,
 				Deployer:     test.deployer,
 				Tagger:       &tag.ChecksumTagger{},
 				watchFactory: test.watcherFactory,
@@ -360,6 +425,7 @@ func TestBuildAndDeployAllArtifacts(t *testing.T) {
 	defer resetClient()
 
 	builder := &TestBuilder{}
+	tester := &TestTester{}
 	deployer := &TestDeployer{}
 	artifacts := []*latest.Artifact{
 		{ImageName: "image1"},
@@ -368,6 +434,7 @@ func TestBuildAndDeployAllArtifacts(t *testing.T) {
 
 	runner := &SkaffoldRunner{
 		Builder:  builder,
+		Tester:   tester,
 		Deployer: deployer,
 		opts:     &config.SkaffoldOptions{},
 	}

From a96cbfbf3931c043edd848fced3f33a77fcd2624 Mon Sep 17 00:00:00 2001
From: Nick Kubala <nkubala@google.com>
Date: Fri, 21 Sep 2018 12:53:09 -0700
Subject: [PATCH 4/9] lint

---
 pkg/skaffold/test/structure/structure.go | 2 +-
 pkg/skaffold/test/structure/types.go     | 6 +++---
 pkg/skaffold/test/types.go               | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/pkg/skaffold/test/structure/structure.go b/pkg/skaffold/test/structure/structure.go
index 6179dfdf5a8..fb1e49aa401 100644
--- a/pkg/skaffold/test/structure/structure.go
+++ b/pkg/skaffold/test/structure/structure.go
@@ -25,7 +25,7 @@ import (
 )
 
 // Test is the entrypoint for running structure tests
-func (tr *TestRunner) Test(image string) error {
+func (tr *Runner) Test(image string) error {
 	logrus.Infof("running structure tests for files %v", tr.testFiles)
 	args := []string{"test", "--image", image}
 	for _, f := range tr.testFiles {
diff --git a/pkg/skaffold/test/structure/types.go b/pkg/skaffold/test/structure/types.go
index af1fb804432..dddf0618b5a 100644
--- a/pkg/skaffold/test/structure/types.go
+++ b/pkg/skaffold/test/structure/types.go
@@ -16,12 +16,12 @@ limitations under the License.
 
 package structure
 
-type TestRunner struct {
+type Runner struct {
 	testFiles []string
 }
 
-func NewStructureTestRunner(files []string) (*TestRunner, error) {
-	return &TestRunner{
+func NewStructureTestRunner(files []string) (*Runner, error) {
+	return &Runner{
 		testFiles: files,
 	}, nil
 }
diff --git a/pkg/skaffold/test/types.go b/pkg/skaffold/test/types.go
index a4938e22f88..524cdf0d1e2 100644
--- a/pkg/skaffold/test/types.go
+++ b/pkg/skaffold/test/types.go
@@ -48,12 +48,12 @@ type FullTester struct {
 // tests runners to run all specified tests on an individual artifact.
 type ArtifactTester struct {
 	ImageName   string
-	TestRunners []TestRunner
+	TestRunners []Runner
 }
 
 // TestRunner is the lowest-level test executor in Skaffold, responsible for
 // running a single test on a single artifact image and returning its result.
 // Any new test type should implement this interface.
-type TestRunner interface {
+type Runner interface {
 	Test(image string) error
 }

From 9c5902914c3d692f41ee9c7298c392b5d40a03b7 Mon Sep 17 00:00:00 2001
From: Nick Kubala <nkubala@google.com>
Date: Mon, 24 Sep 2018 11:30:28 -0700
Subject: [PATCH 5/9] check runner.Tester in unit tests. pin cst binary to
 version in Dockerfile

---
 deploy/skaffold/Dockerfile         | 3 ++-
 pkg/skaffold/runner/runner_test.go | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/deploy/skaffold/Dockerfile b/deploy/skaffold/Dockerfile
index 16370c60ded..bcf4e691eb4 100644
--- a/deploy/skaffold/Dockerfile
+++ b/deploy/skaffold/Dockerfile
@@ -47,7 +47,8 @@ RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8
 RUN apt-get update \
   && apt-get install -y bazel
 
-RUN curl -LO https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 \
+ENV CONTAINER_STRUCTURE_TEST_VERSION=1.5.0
+RUN curl -LO https://storage.googleapis.com/container-structure-test/v${CONTAINER_STRUCTURE_TEST_VERSION}/container-structure-test-linux-amd64 \
   && chmod +x container-structure-test-linux-amd64 \
   && mv container-structure-test-linux-amd64 /usr/local/bin/container-structure-test
 
diff --git a/pkg/skaffold/runner/runner_test.go b/pkg/skaffold/runner/runner_test.go
index 02c6fa6a617..0db39ae2590 100644
--- a/pkg/skaffold/runner/runner_test.go
+++ b/pkg/skaffold/runner/runner_test.go
@@ -240,9 +240,10 @@ func TestNewForConfig(t *testing.T) {
 
 			testutil.CheckError(t, test.shouldErr, err)
 			if cfg != nil {
-				b, _, d := WithTimings(test.expectedBuilder, test.expectedTester, test.expectedDeployer)
+				b, _t, d := WithTimings(test.expectedBuilder, test.expectedTester, test.expectedDeployer)
 
 				testutil.CheckErrorAndTypeEquality(t, test.shouldErr, err, b, cfg.Builder)
+				testutil.CheckErrorAndTypeEquality(t, test.shouldErr, err, _t, cfg.Tester)
 				testutil.CheckErrorAndTypeEquality(t, test.shouldErr, err, d, cfg.Deployer)
 			}
 		})

From f703c8dd806b5ea8a6301a0b12223b6987ddf75a Mon Sep 17 00:00:00 2001
From: Nick Kubala <nkubala@google.com>
Date: Tue, 2 Oct 2018 10:30:57 -0700
Subject: [PATCH 6/9] move config changes to v1alpha4 and examples to
 integration/examples

---
 examples/getting-started/skaffold.yaml                   | 9 ---------
 .../getting-started/profile_structure_test.yaml          | 0
 integration/examples/getting-started/skaffold.yaml       | 9 +++++++++
 .../examples}/getting-started/structure_test.yaml        | 0
 pkg/skaffold/schema/latest/config.go                     | 2 +-
 pkg/skaffold/schema/v1alpha3/config.go                   | 9 ---------
 pkg/skaffold/test/test.go                                | 4 ++--
 7 files changed, 12 insertions(+), 21 deletions(-)
 rename {examples => integration/examples}/getting-started/profile_structure_test.yaml (100%)
 rename {examples => integration/examples}/getting-started/structure_test.yaml (100%)

diff --git a/examples/getting-started/skaffold.yaml b/examples/getting-started/skaffold.yaml
index 48361be1a3e..fb4d4adbc4c 100644
--- a/examples/getting-started/skaffold.yaml
+++ b/examples/getting-started/skaffold.yaml
@@ -3,10 +3,6 @@ kind: Config
 build:
   artifacts:
   - imageName: gcr.io/k8s-skaffold/skaffold-example
-test:
-  - imageName: gcr.io/k8s-skaffold/skaffold-example
-    structureTests:
-      - structure_test.yaml
 deploy:
   kubectl:
     manifests:
@@ -16,8 +12,3 @@ profiles:
     build:
       googleCloudBuild:
         projectId: k8s-skaffold
-  - name: test
-    test:
-      - imageName: gcr.io/k8s-skaffold/skaffold-example
-        structureTests:
-          - profile_structure_test.yaml
diff --git a/examples/getting-started/profile_structure_test.yaml b/integration/examples/getting-started/profile_structure_test.yaml
similarity index 100%
rename from examples/getting-started/profile_structure_test.yaml
rename to integration/examples/getting-started/profile_structure_test.yaml
diff --git a/integration/examples/getting-started/skaffold.yaml b/integration/examples/getting-started/skaffold.yaml
index 785d94052de..77ade13d99b 100644
--- a/integration/examples/getting-started/skaffold.yaml
+++ b/integration/examples/getting-started/skaffold.yaml
@@ -3,6 +3,10 @@ kind: Config
 build:
   artifacts:
   - image: gcr.io/k8s-skaffold/skaffold-example
+test:
+  - image: gcr.io/k8s-skaffold/skaffold-example
+    structureTests:
+      - structure_test.yaml
 deploy:
   kubectl:
     manifests:
@@ -12,3 +16,8 @@ profiles:
     build:
       googleCloudBuild:
         projectId: k8s-skaffold
+  - name: test
+    test:
+      - image: gcr.io/k8s-skaffold/skaffold-example
+        structureTests:
+          - profile_structure_test.yaml
diff --git a/examples/getting-started/structure_test.yaml b/integration/examples/getting-started/structure_test.yaml
similarity index 100%
rename from examples/getting-started/structure_test.yaml
rename to integration/examples/getting-started/structure_test.yaml
diff --git a/pkg/skaffold/schema/latest/config.go b/pkg/skaffold/schema/latest/config.go
index 2c650e754d1..c9f7efbc1a8 100644
--- a/pkg/skaffold/schema/latest/config.go
+++ b/pkg/skaffold/schema/latest/config.go
@@ -118,7 +118,7 @@ type KanikoBuild struct {
 }
 
 // TestCase is a struct containing all the specified test
-// configuration for an image
+// configuration for an image.
 type TestCase struct {
 	ImageName      string   `yaml:"image"`
 	StructureTests []string `yaml:"structureTests,omitempty"`
diff --git a/pkg/skaffold/schema/v1alpha3/config.go b/pkg/skaffold/schema/v1alpha3/config.go
index b02fb52f42a..35cf08cff54 100644
--- a/pkg/skaffold/schema/v1alpha3/config.go
+++ b/pkg/skaffold/schema/v1alpha3/config.go
@@ -34,7 +34,6 @@ type SkaffoldConfig struct {
 	Kind       string `yaml:"kind"`
 
 	Build    BuildConfig  `yaml:"build,omitempty"`
-	Test     []TestCase   `yaml:"test,omitempty"`
 	Deploy   DeployConfig `yaml:"deploy,omitempty"`
 	Profiles []Profile    `yaml:"profiles,omitempty"`
 }
@@ -117,13 +116,6 @@ type KanikoBuild struct {
 	Timeout        string             `yaml:"timeout,omitempty"`
 }
 
-// TestCase is a struct containing all the specified test
-// configuration for an image.
-type TestCase struct {
-	ImageName      string   `yaml:"imageName"`
-	StructureTests []string `yaml:"structureTests,omitempty"`
-}
-
 // DeployConfig contains all the configuration needed by the deploy steps
 type DeployConfig struct {
 	DeployType `yaml:",inline"`
@@ -220,7 +212,6 @@ type Artifact struct {
 type Profile struct {
 	Name   string       `yaml:"name"`
 	Build  BuildConfig  `yaml:"build,omitempty"`
-	Test   []TestCase   `yaml:"test,omitempty"`
 	Deploy DeployConfig `yaml:"deploy,omitempty"`
 }
 
diff --git a/pkg/skaffold/test/test.go b/pkg/skaffold/test/test.go
index bd37056106d..eb3c4a3d55f 100644
--- a/pkg/skaffold/test/test.go
+++ b/pkg/skaffold/test/test.go
@@ -20,7 +20,7 @@ import (
 	"io"
 
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
-	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v1alpha3"
+	latest "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v1alpha4"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/test/structure"
 
 	"github.com/pkg/errors"
@@ -29,7 +29,7 @@ import (
 // NewTester parses the provided test cases from the Skaffold config,
 // and returns a Tester instance with all the necessary test runners
 // to run all specified tests.
-func NewTester(testCases *[]v1alpha3.TestCase) (Tester, error) {
+func NewTester(testCases *[]latest.TestCase) (Tester, error) {
 	testers := []*ArtifactTester{}
 	deps := []string{}
 	for _, testCase := range *testCases {

From 50e565af40d711a01490335a94c38d96341bc4c3 Mon Sep 17 00:00:00 2001
From: Nick Kubala <nkubala@google.com>
Date: Tue, 2 Oct 2018 11:52:02 -0700
Subject: [PATCH 7/9] Fix tests

---
 pkg/skaffold/runner/runner_test.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pkg/skaffold/runner/runner_test.go b/pkg/skaffold/runner/runner_test.go
index 0db39ae2590..7c1ae070a5d 100644
--- a/pkg/skaffold/runner/runner_test.go
+++ b/pkg/skaffold/runner/runner_test.go
@@ -295,15 +295,15 @@ func TestRun(t *testing.T) {
 		},
 		{
 			description: "run test error",
-			config: &v1alpha3.SkaffoldConfig{
-				Build: v1alpha3.BuildConfig{
-					Artifacts: []*v1alpha3.Artifact{
+			config: &latest.SkaffoldConfig{
+				Build: latest.BuildConfig{
+					Artifacts: []*latest.Artifact{
 						{
 							ImageName: "test",
 						},
 					},
 				},
-				Test: []v1alpha3.TestCase{
+				Test: []latest.TestCase{
 					{
 						ImageName:      "test",
 						StructureTests: []string{"fake_file.yaml"},

From 7907f24f7eeb38f4298ebba98713dfa828ff6b61 Mon Sep 17 00:00:00 2001
From: Nick Kubala <nkubala@google.com>
Date: Tue, 2 Oct 2018 14:11:50 -0700
Subject: [PATCH 8/9] fix rebase

---
 pkg/skaffold/runner/runner_test.go | 2 +-
 pkg/skaffold/test/test.go          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/skaffold/runner/runner_test.go b/pkg/skaffold/runner/runner_test.go
index 7c1ae070a5d..2260d97e414 100644
--- a/pkg/skaffold/runner/runner_test.go
+++ b/pkg/skaffold/runner/runner_test.go
@@ -371,7 +371,7 @@ func TestDev(t *testing.T) {
 			tester: &TestTester{
 				errors: []error{fmt.Errorf("")},
 			},
-			watcherFactory: NewWatcherFactory(nil),
+			watcherFactory: NewWatcherFactory(nil, nil),
 			shouldErr:      true,
 		},
 		{
diff --git a/pkg/skaffold/test/test.go b/pkg/skaffold/test/test.go
index eb3c4a3d55f..4c434c25137 100644
--- a/pkg/skaffold/test/test.go
+++ b/pkg/skaffold/test/test.go
@@ -20,7 +20,7 @@ import (
 	"io"
 
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
-	latest "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v1alpha4"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/test/structure"
 
 	"github.com/pkg/errors"

From a7b9412b830bc82ab2b006a4fe4b8e5938e745c0 Mon Sep 17 00:00:00 2001
From: Nick Kubala <nkubala@google.com>
Date: Tue, 2 Oct 2018 15:40:22 -0700
Subject: [PATCH 9/9] add support for wildcards when specifying test files

---
 .../examples/getting-started/skaffold.yaml       |  4 ++--
 .../{ => test}/profile_structure_test.yaml       |  0
 .../{ => test}/structure_test.yaml               |  0
 pkg/skaffold/test/test.go                        | 16 ++++++++++++++--
 4 files changed, 16 insertions(+), 4 deletions(-)
 rename integration/examples/getting-started/{ => test}/profile_structure_test.yaml (100%)
 rename integration/examples/getting-started/{ => test}/structure_test.yaml (100%)

diff --git a/integration/examples/getting-started/skaffold.yaml b/integration/examples/getting-started/skaffold.yaml
index 77ade13d99b..7ace8c5b6f8 100644
--- a/integration/examples/getting-started/skaffold.yaml
+++ b/integration/examples/getting-started/skaffold.yaml
@@ -6,7 +6,7 @@ build:
 test:
   - image: gcr.io/k8s-skaffold/skaffold-example
     structureTests:
-      - structure_test.yaml
+      - ./test/*
 deploy:
   kubectl:
     manifests:
@@ -20,4 +20,4 @@ profiles:
     test:
       - image: gcr.io/k8s-skaffold/skaffold-example
         structureTests:
-          - profile_structure_test.yaml
+          - ./test/profile_structure_test.yaml
diff --git a/integration/examples/getting-started/profile_structure_test.yaml b/integration/examples/getting-started/test/profile_structure_test.yaml
similarity index 100%
rename from integration/examples/getting-started/profile_structure_test.yaml
rename to integration/examples/getting-started/test/profile_structure_test.yaml
diff --git a/integration/examples/getting-started/structure_test.yaml b/integration/examples/getting-started/test/structure_test.yaml
similarity index 100%
rename from integration/examples/getting-started/structure_test.yaml
rename to integration/examples/getting-started/test/structure_test.yaml
diff --git a/pkg/skaffold/test/test.go b/pkg/skaffold/test/test.go
index 4c434c25137..e75861411e0 100644
--- a/pkg/skaffold/test/test.go
+++ b/pkg/skaffold/test/test.go
@@ -18,10 +18,12 @@ package test
 
 import (
 	"io"
+	"os"
 
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/test/structure"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
 
 	"github.com/pkg/errors"
 )
@@ -32,17 +34,27 @@ import (
 func NewTester(testCases *[]latest.TestCase) (Tester, error) {
 	testers := []*ArtifactTester{}
 	deps := []string{}
+	// TODO(nkubala): copied this from runner.getDeployer(), this should be moved somewhere else
+	cwd, err := os.Getwd()
+	if err != nil {
+		return nil, errors.Wrap(err, "finding current directory")
+	}
 	for _, testCase := range *testCases {
 		testRunner := &ArtifactTester{
 			ImageName: testCase.ImageName,
 		}
 		if testCase.StructureTests != nil {
-			stRunner, err := structure.NewStructureTestRunner(testCase.StructureTests)
+			stFiles, err := util.ExpandPathsGlob(cwd, testCase.StructureTests)
+			if err != nil {
+				return FullTester{}, errors.Wrap(err, "expanding test file paths")
+			}
+			stRunner, err := structure.NewStructureTestRunner(stFiles)
 			if err != nil {
 				return FullTester{}, errors.Wrap(err, "retrieving structure test runner")
 			}
 			testRunner.TestRunners = append(testRunner.TestRunners, stRunner)
-			deps = append(deps, testCase.StructureTests...)
+
+			deps = append(deps, stFiles...)
 		}
 		testers = append(testers, testRunner)
 	}