From 7f68d01221b6f6289d7acac3f5ab85dfed17060a Mon Sep 17 00:00:00 2001 From: spacetj Date: Mon, 9 Dec 2019 17:33:00 +1100 Subject: [PATCH] add relative url from test file (#12) --- cmd/test.go | 5 +-- main.go | 4 -- pkg/kubetest/assertions.go | 4 +- pkg/kubetest/common.go | 89 -------------------------------------- pkg/kubetest/kubetest.go | 42 +++++++++++------- pkg/kubetest/resources.go | 18 +++++--- pkg/kubetest/testresult.go | 27 +++++++++--- pkg/kubetest/types.go | 9 ++-- samples/kube-test.yaml | 12 ++--- 9 files changed, 75 insertions(+), 135 deletions(-) diff --git a/cmd/test.go b/cmd/test.go index 98c19de..6a790a9 100644 --- a/cmd/test.go +++ b/cmd/test.go @@ -54,7 +54,7 @@ func init() { testCmd.Flags().StringP("kubeconfig", "c", defaultKubeConfig, "Path to kubeconfig file.") testCmd.Flags().StringP("kind", "k", "kubetest", "Kind of test, only kubetest is supported at the moment.") testCmd.Flags().Float64P("threshold", "t", 80, "Percentage of tests to pass, else return failure.") - testCmd.Flags().IntP("verbose", "v", 5, "Verbosity Level, choose between 1 being Fatal - 7 being .") + testCmd.Flags().IntP("verbose", "v", 4, "Verbosity Level, choose between 1 being Fatal - 7 being .") testCmd.Flags().BoolP("incluster", "i", false, "Get kubeconfig from in cluster.") } @@ -62,7 +62,7 @@ func testExecute(cmd *cobra.Command, args []string) { verbosity, err := cmd.Flags().GetInt("verbose") if err != nil { logrus.WithFields(logrus.Fields{"flag": "verbose"}).Error("Unable to parse flag. Defaulting to INFO.") - verbosity = 5 + verbosity = 4 } log := &logging.Logger{} log.SetVerbosity(verbosity) @@ -105,5 +105,4 @@ func testExecute(cmd *cobra.Command, args []string) { } log.Info("kind", "kubetest", "Finished Tests") } - } diff --git a/main.go b/main.go index 8d9de8c..6ff1f14 100644 --- a/main.go +++ b/main.go @@ -22,8 +22,6 @@ THE SOFTWARE. package main import ( - "fmt" - "github.com/covarity/anchorctl/cmd" ) @@ -33,7 +31,5 @@ var ( ) func main() { - fmt.Println("Version: " + Version) - fmt.Println("Build: " + Build) cmd.Execute() } diff --git a/pkg/kubetest/assertions.go b/pkg/kubetest/assertions.go index edaed68..ce74091 100644 --- a/pkg/kubetest/assertions.go +++ b/pkg/kubetest/assertions.go @@ -62,7 +62,7 @@ func assertJSONPath(obj interface{}, path, value string) (bool, error) { func (av *validationTest) test(res resource) (bool, error) { - _, _, err := res.Manifest.apply(av.client) + _, _, err := res.Manifest.apply(av.client, true) if err != nil && err.Error() == av.ExpectedError { log.InfoWithFields(map[string]interface{}{ "test": "AssertValidation", @@ -89,7 +89,7 @@ func (am *mutationTest) test(res resource) (bool, error) { return false, fmt.Errorf("Invalid Manifest to apply") } - _, obj, err := res.Manifest.apply(am.client) + _, obj, err := res.Manifest.apply(am.client, false) if err != nil { return false, err } diff --git a/pkg/kubetest/common.go b/pkg/kubetest/common.go index f736e55..93a9077 100644 --- a/pkg/kubetest/common.go +++ b/pkg/kubetest/common.go @@ -6,13 +6,8 @@ import ( "reflect" "gopkg.in/yaml.v2" - v1 "k8s.io/api/core/v1" - "k8s.io/api/extensions/v1beta1" - - appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -106,87 +101,3 @@ func getSlice(t interface{}) []interface{} { return slicedInterface } - -// ApplyFile mimics kubectl apply -f. Takes in a path to a file and applies that object to the cluster and returns the applied object. -func applyAction(client *kubernetes.Clientset, pathToFile, action string) (*kubeMetadata, interface{}, error) { - - decode := scheme.Codecs.UniversalDeserializer().Decode - - bytes, err := ioutil.ReadFile(pathToFile) - objectMetadata := &kubeMetadata{} - var object interface{} - - if err != nil { - log.Fatal(err, "Error reading the test file.") - } - - obj, _, err := decode(bytes, nil, nil) - - if err != nil { - log.Fatal(err, "Error decoding KubeTest object.") - } - - err = yaml.Unmarshal(bytes, &objectMetadata) - - if err != nil { - log.Fatal(err, "Error while unmarshalling KubeTest Metadata.") - } - - log.InfoWithFields(map[string]interface{}{ - "action": action, - "name": objectMetadata.Metadata.Name, - "namespace": objectMetadata.Metadata.Namespace, - }, "Applying action to file") - - switch obj.(type) { - case *appsv1.Deployment: - deploy := obj.(*appsv1.Deployment) - if action == "CREATE" { - object, err = client.AppsV1().Deployments(objectMetadata.Metadata.Namespace).Create(deploy) - } else if action == "UPDATE" { - object, err = client.AppsV1().Deployments(objectMetadata.Metadata.Namespace).Update(deploy) - } else { - err = client.AppsV1().Deployments(objectMetadata.Metadata.Namespace).Delete(objectMetadata.Metadata.Name, &metav1.DeleteOptions{}) - } - case *v1.Pod: - pod := obj.(*v1.Pod) - if action == "CREATE" { - object, err = client.CoreV1().Pods(objectMetadata.Metadata.Namespace).Create(pod) - } else if action == "UPDATE" { - object, err = client.CoreV1().Pods(objectMetadata.Metadata.Namespace).Update(pod) - } else { - err = client.CoreV1().Pods(objectMetadata.Metadata.Namespace).Delete(objectMetadata.Metadata.Name, &metav1.DeleteOptions{}) - } - case *v1.Service: - service := obj.(*v1.Service) - if action == "CREATE" { - object, err = client.CoreV1().Services(objectMetadata.Metadata.Namespace).Create(service) - } else if action == "UPDATE" { - object, err = client.CoreV1().Services(objectMetadata.Metadata.Namespace).Update(service) - } else { - err = client.CoreV1().Services(objectMetadata.Metadata.Namespace).Delete(objectMetadata.Metadata.Name, &metav1.DeleteOptions{}) - } - case *v1beta1.Ingress: - ingress := obj.(*v1beta1.Ingress) - if action == "CREATE" { - object, err = client.ExtensionsV1beta1().Ingresses(objectMetadata.Metadata.Namespace).Create(ingress) - } else if action == "UPDATE" { - object, err = client.ExtensionsV1beta1().Ingresses(objectMetadata.Metadata.Namespace).Update(ingress) - } else { - err = client.ExtensionsV1beta1().Ingresses(objectMetadata.Metadata.Namespace).Delete(objectMetadata.Metadata.Name, &metav1.DeleteOptions{}) - } - case *v1beta1.DaemonSet: - ds := obj.(*v1beta1.DaemonSet) - if action == "CREATE" { - object, err = client.ExtensionsV1beta1().DaemonSets(objectMetadata.Metadata.Namespace).Create(ds) - } else if action == "UPDATE" { - object, err = client.ExtensionsV1beta1().DaemonSets(objectMetadata.Metadata.Namespace).Update(ds) - } else { - err = client.ExtensionsV1beta1().DaemonSets(objectMetadata.Metadata.Namespace).Delete(objectMetadata.Metadata.Name, &metav1.DeleteOptions{}) - } - default: - object, err = nil, fmt.Errorf("ApplyAction for kind is not implemented") - } - - return objectMetadata, object, err -} diff --git a/pkg/kubetest/kubetest.go b/pkg/kubetest/kubetest.go index 6fffcaa..97b77ed 100644 --- a/pkg/kubetest/kubetest.go +++ b/pkg/kubetest/kubetest.go @@ -13,6 +13,7 @@ var requiredField = map[string][]string{ } var log *logging.Logger +var testFilePath string func Assert(logger *logging.Logger, threshold float64, incluster bool, kubeconfig, testfile string) { @@ -22,11 +23,12 @@ func Assert(logger *logging.Logger, threshold float64, incluster bool, kubeconfi if err != nil { log.Fatal(err, "Unable to get kubernetes client") } - - kubeTest, err := decodeTestFile(client, testfile) + testFilePath = testfile + kubeTest, err := decodeTestFile(client, testFilePath) if err != nil { log.Fatal(err, "Unable to decode test file") } + kubeTest.testFilePath = testfile executeLifecycle(kubeTest.Spec.Lifecycle.PostStart, client) @@ -42,56 +44,64 @@ func Assert(logger *logging.Logger, threshold float64, incluster bool, kubeconfi func runTests(client *kubernetes.Clientset, kubeTest *kubeTest) *testResult { res := testResult{} - for _, i := range kubeTest.Spec.Tests { - switch i.Type { + for i, test := range kubeTest.Spec.Tests { + switch test.Type { case "AssertJSONPath": var jsonTestObj *jsonTest - err := mapstructure.Decode(i.Spec, &jsonTestObj) + res.addResultToRow(i, "AssertJSONPath") + err := mapstructure.Decode(test.Spec, &jsonTestObj) + res.addResultToRow(i, "JSONPath: "+jsonTestObj.JsonPath+" Value: "+jsonTestObj.Value) if err != nil { res.invalid++ + res.addResultToRow(i, "❌") log.Warn("Error", err.Error(), "Decoding AssertJSONPath returned error") continue } - jsonTestObj.client = client - runKubeTester(jsonTestObj, i, &res) + runKubeTester(jsonTestObj, test, &res, i) case "AssertValidation": var validationTest *validationTest - err := mapstructure.Decode(i.Spec, &validationTest) + res.addResultToRow(i, "AssertValidation") + err := mapstructure.Decode(test.Spec, &validationTest) + res.addResultToRow(i, "ExpectedError: "+validationTest.ExpectedError[:25]+"...") if err != nil { res.invalid++ + res.addResultToRow(i, "❌") log.Warn("Error", err.Error(), "Decoding AssertValidation returned error") continue } - validationTest.client = client - runKubeTester(validationTest, i, &res) + runKubeTester(validationTest, test, &res, i) case "AssertMutation": var mutationTest *mutationTest - err := mapstructure.Decode(i.Spec, &mutationTest) + res.addResultToRow(i, "AssertMutation") + err := mapstructure.Decode(test.Spec, &mutationTest) + res.addResultToRow(i, "JSONPath: "+mutationTest.JsonPath+" Value "+mutationTest.Value) if err != nil { res.invalid++ + res.addResultToRow(i, "❌") log.Warn("Error", err.Error(), "Decoding AssertMutation returned error") continue } - mutationTest.client = client - runKubeTester(mutationTest, i, &res) - + runKubeTester(mutationTest, test, &res, i) } } return &res } -func runKubeTester(kubetester kubeTester, i test, res *testResult) { - if result, err := kubetester.test(i.Resource); err != nil { +func runKubeTester(kubetester kubeTester, test test, res *testResult, i int) { + if result, err := kubetester.test(test.Resource); err != nil { + res.addResultToRow(i, "❌") res.invalid++ } else if result == false { + res.addResultToRow(i, "❌") res.failed++ } else { + res.addResultToRow(i, "✅") res.passed++ } } diff --git a/pkg/kubetest/resources.go b/pkg/kubetest/resources.go index baecb9c..7df8f16 100644 --- a/pkg/kubetest/resources.go +++ b/pkg/kubetest/resources.go @@ -2,15 +2,15 @@ package kubetest import ( "fmt" - "io/ioutil" - "gopkg.in/yaml.v2" + "io/ioutil" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" + "path/filepath" ) type resource struct { @@ -30,7 +30,7 @@ type lifecycle struct { func executeLifecycle(manifests []manifest, client *kubernetes.Clientset) { for _, i := range manifests { - _, _, err := i.apply(client) + _, _, err := i.apply(client, false) if err != nil { log.Fatal(err, "Failed Lifecycle steps") } @@ -102,7 +102,7 @@ func (mf manifest) valid() bool { } // ApplyFile mimics kubectl apply -f. Takes in a path to a file and applies that object to the cluster and returns the applied object. -func (mf manifest) apply(client *kubernetes.Clientset) (*kubeMetadata, interface{}, error) { +func (mf manifest) apply(client *kubernetes.Clientset, expectError bool) (*kubeMetadata, interface{}, error) { if valid := mf.valid(); valid != true { return nil, nil, fmt.Errorf("Invalid Manifest to apply") @@ -111,8 +111,14 @@ func (mf manifest) apply(client *kubernetes.Clientset) (*kubeMetadata, interface decode := scheme.Codecs.UniversalDeserializer().Decode objectMetadata := &kubeMetadata{} var object interface{} + var filePath string + if testFilePath != "" { + filePath = filepath.Dir(testFilePath) + "/" + mf.Path + } else { + filePath = mf.Path + } - bytes, err := ioutil.ReadFile(mf.Path) + bytes, err := ioutil.ReadFile(filePath) if err != nil { log.Error(err, "Error reading the "+mf.Path+" file.") return nil, nil, err @@ -194,7 +200,7 @@ func (mf manifest) apply(client *kubernetes.Clientset) (*kubeMetadata, interface object, err = nil, fmt.Errorf("ApplyAction for kind is not implemented") } - if err != nil { + if err != nil && expectError != true { log.WarnWithFields(map[string]interface{}{ "Stage": "PostStart", "Path": mf.Path, diff --git a/pkg/kubetest/testresult.go b/pkg/kubetest/testresult.go index 55c49ae..e5da2ae 100644 --- a/pkg/kubetest/testresult.go +++ b/pkg/kubetest/testresult.go @@ -10,9 +10,19 @@ import ( type testResult struct { invalid, passed, failed, total int successRatio, threshold float64 + testRuns [][]string } func (res *testResult) print() { + + testList := tablewriter.NewWriter(os.Stdout) + testList.SetHeader([]string{"Test Type", "Spec", "Passed"}) + testList.SetBorder(false) + testList.AppendBulk(res.testRuns) + fmt.Println() + testList.Render() + fmt.Println() + res.successRatio = res.calculateSuccessRatio() data := [][]string{ @@ -23,15 +33,22 @@ func (res *testResult) print() { {"Expected Coverage", fmt.Sprintf("%.2f", res.threshold)}, {"Actual Coverage", fmt.Sprintf("%.2f", res.successRatio)}, } - table := tablewriter.NewWriter(os.Stdout) - table.SetHeader([]string{"Tests", "Number"}) - table.SetBorder(false) - table.AppendBulk(data) + testSumamry := tablewriter.NewWriter(os.Stdout) + testSumamry.SetHeader([]string{"Tests", "Number"}) + testSumamry.SetBorder(false) + testSumamry.AppendBulk(data) fmt.Println() - table.Render() + testSumamry.Render() fmt.Println() } +func (res *testResult) addResultToRow(row int, add string) { + if res.testRuns == nil { + res.testRuns = make([][]string, 10) + } + res.testRuns[row] = append(res.testRuns[row], add) +} + func (res *testResult) validate() { if res.successRatio < res.threshold { log.Fatal(fmt.Errorf("Expected %.2f, Got %.2f", res.threshold, res.successRatio), "Failed Test Threshold") diff --git a/pkg/kubetest/types.go b/pkg/kubetest/types.go index 26d76e8..1dd0f03 100644 --- a/pkg/kubetest/types.go +++ b/pkg/kubetest/types.go @@ -5,10 +5,11 @@ import ( ) type kubeTest struct { - ApiVersion string `yaml:"apiVersion"` - Kind string - Metadata metadata - Spec kubeTestSpec + ApiVersion string `yaml:"apiVersion"` + Kind string + Metadata metadata + Spec kubeTestSpec + testFilePath string } type kubeTestSpec struct { diff --git a/samples/kube-test.yaml b/samples/kube-test.yaml index aa6abc2..3e62302 100644 --- a/samples/kube-test.yaml +++ b/samples/kube-test.yaml @@ -5,14 +5,14 @@ metadata: spec: lifecycle: postStart: - - path: "./samples/fixtures/applications-ns.yaml" + - path: "./fixtures/applications-ns.yaml" action: "CREATE" - - path: "./samples/fixtures/hello-world-nginx.yaml" + - path: "./fixtures/hello-world-nginx.yaml" action: "CREATE" preStop: - - path: "./samples/fixtures/hello-world-nginx.yaml" + - path: "./fixtures/hello-world-nginx.yaml" action: "DELETE" - - path: "./samples/fixtures/applications-ns.yaml" + - path: "./fixtures/applications-ns.yaml" action: "DELETE" tests: - type: AssertJSONPath @@ -32,7 +32,7 @@ spec: expectedError: "Internal error occurred: admission webhook \"webhook.openpolicyagent.org\" denied the request: External Loadbalancers cannot be deployed in this cluster" resource: manifest: - path: "./samples/fixtures/loadbalancer.yaml" + path: "./fixtures/loadbalancer.yaml" action: CREATE - type: AssertMutation spec: @@ -40,5 +40,5 @@ spec: value: "workload" resource: manifest: - path: "./samples/fixtures/deploy.yaml" + path: "./fixtures/deploy.yaml" action: CREATE