feat: add test yaml config (#986)

* feat: add test yaml config

* switch to test configs in test dirs

* add skip methods to blueprint test

* embed BlueprintTestConfig in BlueprintTests

infra/blueprint-test/pkg/discovery/config.go

@@ -0,0 +1,59 @@
+package discovery
+
+import (
+	"fmt"
+	"os"
+
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+)
+
+const (
+	DefaultTestConfigFilename = "test.yaml"
+	blueprintTestKind         = "BlueprintTest"
+	blueprintTestAPIVersion   = "blueprints.cloud.google.com/v1alpha1"
+)
+
+type BlueprintTestConfig struct {
+	yaml.ResourceMeta `json:",inline" yaml:",inline"`
+	Spec              struct {
+		Skip bool `json:"skip" yaml:"skip"`
+	} `json:"spec" yaml:"spec"`
+	Path string
+}
+
+// GetTestConfig returns BlueprintTestConfig if found
+func GetTestConfig(path string) (BlueprintTestConfig, error) {
+	_, err := os.Stat(path)
+	if os.IsNotExist(err) {
+		// BlueprintTestConfig does not exist, so we return an equivalent empty config
+		emptyCfg := BlueprintTestConfig{}
+		emptyCfg.Spec.Skip = false
+		return emptyCfg, nil
+	}
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return BlueprintTestConfig{}, fmt.Errorf("error reading %s: %v", path, err)
+	}
+	var bpc BlueprintTestConfig
+	err = yaml.Unmarshal(data, &bpc)
+	if err != nil {
+		return BlueprintTestConfig{}, fmt.Errorf("error unmarshalling %s: %v", data, err)
+	}
+	bpc.Path = path
+	err = isValidTestConfig(bpc)
+	if err != nil {
+		return BlueprintTestConfig{}, fmt.Errorf("error validating testconfig in %s: %v", path, err)
+	}
+	return bpc, nil
+}
+
+// isValidTestConfig validates a given BlueprintTestConfig
+func isValidTestConfig(b BlueprintTestConfig) error {
+	if b.APIVersion != blueprintTestAPIVersion {
+		return fmt.Errorf("invalid APIVersion %s expected %s", b.APIVersion, blueprintTestAPIVersion)
+	}
+	if b.Kind != blueprintTestKind {
+		return fmt.Errorf("invalid Kind %s expected %s", b.Kind, blueprintTestKind)
+	}
+	return nil
+}

infra/blueprint-test/pkg/discovery/config_test.go

@@ -0,0 +1,75 @@
+package discovery
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_ShouldSkipTest(t *testing.T) {
+	tests := []struct {
+		name       string
+		testCfg    string
+		shouldSkip bool
+		errMsg     string
+	}{
+		{
+			name: "simple",
+			testCfg: `apiVersion: blueprints.cloud.google.com/v1alpha1
+kind: BlueprintTest
+metadata:
+  name: test
+spec:
+  skip: true
+`,
+			shouldSkip: true,
+		},
+		{
+			name: "invalid",
+			testCfg: `apiVersion: blueprints.cloud.google.com/v1alpha1
+kind: foo
+metadata:
+  name: test
+spec:
+  skip: true
+`,
+			errMsg: "invalid Kind foo expected BlueprintTest",
+		},
+		{
+			name:       "empty none skipped",
+			shouldSkip: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert := assert.New(t)
+			testCfgPath := ""
+			if tt.testCfg != "" {
+				testCfgPath = setupTestCfg(t, tt.testCfg)
+			}
+			defer os.RemoveAll(testCfgPath)
+			bpTestCfg, err := GetTestConfig(testCfgPath)
+			if tt.errMsg != "" {
+				assert.NotNil(err)
+				assert.Contains(err.Error(), tt.errMsg)
+			} else {
+				assert.NoError(err)
+				assert.Equal(tt.shouldSkip, bpTestCfg.Spec.Skip)
+			}
+		})
+	}
+}
+
+func setupTestCfg(t *testing.T, data string) string {
+	t.Helper()
+	assert := assert.New(t)
+	baseDir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	fPath := path.Join(baseDir, "test.yaml")
+	err = ioutil.WriteFile(fPath, []byte(data), 0644)
+	assert.NoError(err)
+	return fPath
+}

infra/blueprint-test/pkg/discovery/discover.go

@@ -53,7 +53,7 @@ func GetConfigDirFromTestDir(cwd string) (string, error) {
 
 // FindTestConfigs attempts to auto discover configs to test and is expected to be executed from a directory containing explicit integration tests.
 // Order of discovery is all explicit tests, followed by all fixtures that do not have explicit tests, followed by all examples that do not have fixtures nor explicit tests.
-func FindTestConfigs(t testing.TB, intTestDir string) []string {
+func FindTestConfigs(t testing.TB, intTestDir string) map[string]string {
 	testBase := intTestDir
 	examplesBase := path.Join(testBase, "../../", ExamplesDir)
 	fixturesBase := path.Join(testBase, "../", FixtureDir)
@@ -69,21 +69,25 @@ func FindTestConfigs(t testing.TB, intTestDir string) []string {
 	if err != nil {
 		t.Logf("Error discovering examples: %v", err)
 	}
-	testCases := make([]string, 0)
-	//TODO(bharathkkb): add overrides
+	testCases := make(map[string]string)
+
 	// if a fixture exists but no explicit test defined
 	for n := range fixtures {
 		_, ok := explicitTests[n]
 		if !ok {
-			testCases = append(testCases, path.Join(fixturesBase, n))
+			testDir := path.Join(fixturesBase, n)
+			testName := fmt.Sprintf("%s/%s", path.Base(path.Dir(testDir)), path.Base(testDir))
+			testCases[testName] = testDir
 		}
 	}
 	// if an example exists that does not have a fixture nor explicit test defined
 	for n := range examples {
 		_, okTest := explicitTests[n]
 		_, okFixture := fixtures[n]
 		if !okTest && !okFixture {
-			testCases = append(testCases, path.Join(examplesBase, n))
+			testDir := path.Join(examplesBase, n)
+			testName := fmt.Sprintf("%s/%s", path.Base(path.Dir(testDir)), path.Base(testDir))
+			testCases[testName] = testDir
 		}
 	}
 	// explicit tests in integration/test_name are not gathered since they are invoked directly

infra/blueprint-test/pkg/krmt/krm.go

@@ -24,20 +24,21 @@ var CommonSetters = []string{"PROJECT_ID", "BILLING_ACCOUNT_ID", "ORG_ID"}
 
 // KRMBlueprintTest implements bpt.Blueprint and stores information associated with a KRM blueprint test.
 type KRMBlueprintTest struct {
-	name         string                   // descriptive name for the test
-	exampleDir   string                   // directory containing KRM blueprint example
-	buildDir     string                   // directory to hydrated blueprint configs pre apply
-	kpt          *kpt.CmdCfg              // kpt cmd config
-	timeout      string                   // timeout for KRM resource status
-	setters      map[string]string        // additional setters to populate
-	updatePkgs   bool                     // whether to update packages in exampleDir
-	updateCommit string                   // specific commit to update to
-	logger       *logger.Logger           // custom logger
-	t            testing.TB               // TestingT or TestingB
-	init         func(*assert.Assertions) // init function
-	apply        func(*assert.Assertions) // apply function
-	verify       func(*assert.Assertions) // verify function
-	teardown     func(*assert.Assertions) // teardown function
+	discovery.BlueprintTestConfig                          // additional blueprint test configs
+	name                          string                   // descriptive name for the test
+	exampleDir                    string                   // directory containing KRM blueprint example
+	buildDir                      string                   // directory to hydrated blueprint configs pre apply
+	kpt                           *kpt.CmdCfg              // kpt cmd config
+	timeout                       string                   // timeout for KRM resource status
+	setters                       map[string]string        // additional setters to populate
+	updatePkgs                    bool                     // whether to update packages in exampleDir
+	updateCommit                  string                   // specific commit to update to
+	logger                        *logger.Logger           // custom logger
+	t                             testing.TB               // TestingT or TestingB
+	init                          func(*assert.Assertions) // init function
+	apply                         func(*assert.Assertions) // apply function
+	verify                        func(*assert.Assertions) // verify function
+	teardown                      func(*assert.Assertions) // teardown function
 }
 
 type krmtOption func(*KRMBlueprintTest)
@@ -129,6 +130,12 @@ func NewKRMBlueprintTest(t testing.TB, opts ...krmtOption) *KRMBlueprintTest {
 		}
 		krmt.exampleDir = exampleDir
 	}
+	// discover test config
+	var err error
+	krmt.BlueprintTestConfig, err = discovery.GetTestConfig(path.Join(krmt.exampleDir, discovery.DefaultTestConfigFilename))
+	if err != nil {
+		t.Fatal(err)
+	}
 	// if no explicit build directory is provided, setup build directory
 	if krmt.buildDir == "" {
 		krmt.buildDir = krmt.getDefaultBuildDir()
@@ -265,12 +272,15 @@ func (b *KRMBlueprintTest) DefaultTeardown(assert *assert.Assertions) {
 	b.kpt.RunCmd("live", "status", "--output", "json", "--poll-until", "deleted", "--timeout", b.timeout)
 }
 
+// ShouldSkip checks if a test should be skipped
+func (b *KRMBlueprintTest) ShouldSkip() bool {
+	return b.Spec.Skip
+}
+
 // AutoDiscoverAndTest discovers KRM config from examples/fixtures and runs tests.
 func AutoDiscoverAndTest(t *gotest.T) {
 	configs := discovery.FindTestConfigs(t, "./")
-	for _, dir := range configs {
-		// dir must be of the form ../fixture/name or ../../examples/name
-		testName := fmt.Sprintf("test-%s-%s", path.Base(path.Dir(dir)), path.Base(dir))
+	for testName, dir := range configs {
 		t.Run(testName, func(t *gotest.T) {
 			nt := NewKRMBlueprintTest(t, WithDir(dir))
 			nt.Test()
@@ -320,6 +330,10 @@ func (b *KRMBlueprintTest) Teardown(assert *assert.Assertions) {
 
 // Test runs init, apply, verify, teardown in order for the blueprint.
 func (b *KRMBlueprintTest) Test() {
+	if b.ShouldSkip() {
+		b.logger.Logf(b.t, "Skipping test due to config %s", b.Path)
+		return
+	}
 	a := assert.New(b.t)
 	// run stages
 	utils.RunStage("init", func() { b.Init(a) })

infra/blueprint-test/pkg/tft/terraform.go

@@ -34,17 +34,18 @@ import (
 
 // TFBlueprintTest implements bpt.Blueprint and stores information associated with a Terraform blueprint test.
 type TFBlueprintTest struct {
-	name      string                   // descriptive name for the test
-	tfDir     string                   // directory containing Terraform configs
-	tfEnvVars map[string]string        // variables to pass to Terraform as environment variables prefixed with TF_VAR_
-	setupDir  string                   // optional directory containing applied TF configs to import outputs as variables for the test
-	vars      map[string]interface{}   // variables to pass to Terraform as flags
-	logger    *logger.Logger           // custom logger
-	t         testing.TB               // TestingT or TestingB
-	init      func(*assert.Assertions) // init function
-	apply     func(*assert.Assertions) // apply function
-	verify    func(*assert.Assertions) // verify function
-	teardown  func(*assert.Assertions) // teardown function
+	discovery.BlueprintTestConfig                          // additional blueprint test configs
+	name                          string                   // descriptive name for the test
+	tfDir                         string                   // directory containing Terraform configs
+	tfEnvVars                     map[string]string        // variables to pass to Terraform as environment variables prefixed with TF_VAR_
+	setupDir                      string                   // optional directory containing applied TF configs to import outputs as variables for the test
+	vars                          map[string]interface{}   // variables to pass to Terraform as flags
+	logger                        *logger.Logger           // custom logger
+	t                             testing.TB               // TestingT or TestingB
+	init                          func(*assert.Assertions) // init function
+	apply                         func(*assert.Assertions) // apply function
+	verify                        func(*assert.Assertions) // verify function
+	teardown                      func(*assert.Assertions) // teardown function
 }
 
 type tftOption func(*TFBlueprintTest)
@@ -133,6 +134,12 @@ func NewTFBlueprintTest(t testing.TB, opts ...tftOption) *TFBlueprintTest {
 		}
 		tft.tfDir = tfdir
 	}
+	// discover test config
+	var err error
+	tft.BlueprintTestConfig, err = discovery.GetTestConfig(path.Join(tft.tfDir, discovery.DefaultTestConfigFilename))
+	if err != nil {
+		t.Fatal(err)
+	}
 	// setupDir is empty, try known setupDir paths
 	if tft.setupDir == "" {
 		setupDir, err := discovery.GetKnownDirInParents(discovery.SetupDir, 2)
@@ -200,12 +207,15 @@ func loadTFEnvVar(m map[string]string, new map[string]string) {
 	}
 }
 
+// ShouldSkip checks if a test should be skipped
+func (b *TFBlueprintTest) ShouldSkip() bool {
+	return b.Spec.Skip
+}
+
 // AutoDiscoverAndTest discovers TF config from examples/fixtures and runs tests.
 func AutoDiscoverAndTest(t *gotest.T) {
 	configs := discovery.FindTestConfigs(t, "./")
-	for _, dir := range configs {
-		// dir must be of the form ../fixture/name or ../../examples/name
-		testName := fmt.Sprintf("test-%s-%s", path.Base(path.Dir(dir)), path.Base(dir))
+	for testName, dir := range configs {
 		t.Run(testName, func(t *gotest.T) {
 			nt := NewTFBlueprintTest(t, WithTFDir(dir))
 			nt.Test()
@@ -285,6 +295,10 @@ func (b *TFBlueprintTest) Teardown(assert *assert.Assertions) {
 
 // Test runs init, apply, verify, teardown in order for the blueprint.
 func (b *TFBlueprintTest) Test() {
+	if b.ShouldSkip() {
+		b.logger.Logf(b.t, "Skipping test due to config %s", b.Path)
+		return
+	}
 	a := assert.New(b.t)
 	// run stages
 	utils.RunStage("init", func() { b.Init(a) })