From 00e85a5de9c703b0f7383d146b953777fdf2b8fb Mon Sep 17 00:00:00 2001 From: Ashish Khatkar Date: Thu, 27 Feb 2020 16:31:39 +0000 Subject: [PATCH 1/4] Adding support to get Yaml for k8s object --- go.mod | 2 +- pkg/hashing/hashing.go | 46 +++-------- pkg/utils/helper.go | 74 +++++++++++++++++ pkg/utils/helper_test.go | 173 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 258 insertions(+), 37 deletions(-) create mode 100644 pkg/utils/helper.go create mode 100644 pkg/utils/helper_test.go diff --git a/go.mod b/go.mod index 6f4a0bf..17b08e8 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( github.com/googleapis/gnostic v0.3.1 // indirect github.com/hashicorp/golang-lru v0.5.3 // indirect github.com/imdario/mergo v0.3.8 // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/pborman/uuid v1.2.0 // indirect github.com/pkg/errors v0.8.1 @@ -23,6 +22,7 @@ require ( golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.2.2 k8s.io/api v0.0.0-20190313235455-40a48860b5ab k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1 k8s.io/client-go v11.0.0+incompatible diff --git a/pkg/hashing/hashing.go b/pkg/hashing/hashing.go index b2f232c..578d094 100644 --- a/pkg/hashing/hashing.go +++ b/pkg/hashing/hashing.go @@ -3,50 +3,24 @@ package hashing import ( "encoding/json" "fmt" + utils "github.com/Yelp/paasta-tools-go/pkg/utils" "hash/fnv" "k8s.io/apimachinery/pkg/util/rand" "reflect" ) func ComputeHashForKubernetesObject(object interface{}) (string, error) { - // By marshaling/unmarshaling the object via JSON we're copying it into a - // map, which is easier to manipulate in generic way than structs. - if b, err := json.Marshal(object); err != nil { - return "", fmt.Errorf("Error while encoding %+v into JSON: %s", object, err) + if m, err := utils.GetHashObjectOfKubernetes(object); err != nil { + return "", err } else { - var v map[string]interface{} - if err := json.Unmarshal(b, &v); err != nil { - return "", fmt.Errorf("Error while decoding JSON %s into an object: %s", v, err) + // By using serialized JSON for hashing we're making the hashing process + // a bit easier (like having maps always being sorted by keys). + if b, err := json.Marshal(m); err != nil { + return "", fmt.Errorf("Error while encoding %+v into JSON: %s", m, err) } else { - // We need only kind/version/spec and labels excluding the label with the - // current hash value while calculating the hash. Also Kubernetes adds - // its own info into `metadata` which we need to ignore. - meta := v["metadata"].(map[string]interface{}) - labels := meta["labels"] - if labels != nil { - delete(labels.(map[string]interface{}), "yelp.com/operator_config_hash") - } else { - labels = make(map[string]interface{}) - } - m := map[string]interface{}{ - "kind": v["kind"], - "apiVersion": v["apiVersion"], - "spec": v["spec"], - "metadata": map[string]interface{}{ - "name": meta["name"], - "namespace": meta["namespace"], - "labels": labels, - }, - } - // By using serialized JSON for hashing we're making the hashing process - // a bit easier (like having maps always being sorted by keys). - if b, err := json.Marshal(m); err != nil { - return "", fmt.Errorf("Error while encoding %+v into JSON: %s", m, err) - } else { - hasher := fnv.New32a() - hasher.Write(b) - return rand.SafeEncodeString(fmt.Sprint(hasher.Sum32())), nil - } + hasher := fnv.New32a() + hasher.Write(b) + return rand.SafeEncodeString(fmt.Sprint(hasher.Sum32())), nil } } } diff --git a/pkg/utils/helper.go b/pkg/utils/helper.go new file mode 100644 index 0000000..1999c6d --- /dev/null +++ b/pkg/utils/helper.go @@ -0,0 +1,74 @@ +package utils + +import ( + "encoding/json" + "fmt" + "gopkg.in/yaml.v2" +) + +// function to get the relevant information out of k8s object that is needed to compute hash +func GetHashObjectOfKubernetes(object interface{}) (map[string]interface{}, error) { + // By marshaling/unmarshaling the object via JSON we're copying it into a + // map, which is easier to manipulate in generic way than structs. + if b, err := json.Marshal(object); err != nil { + return nil, fmt.Errorf("Error while encoding %+v into JSON: %s", object, err) + } else { + var v map[string]interface{} + if err := json.Unmarshal(b, &v); err != nil { + return nil, fmt.Errorf("Error while decoding JSON %s into an object: %s", v, err) + } else { + // We need only kind/version/spec and labels excluding the label with the + // current hash value while calculating the hash. Also Kubernetes adds + // its own info into `metadata` which we need to ignore. + meta := v["metadata"].(map[string]interface{}) + labels := meta["labels"] + if labels != nil { + delete(labels.(map[string]interface{}), "yelp.com/operator_config_hash") + } else { + labels = make(map[string]interface{}) + } + m := map[string]interface{}{ + "kind": v["kind"], + "apiVersion": v["apiVersion"], + "spec": v["spec"], + "metadata": map[string]interface{}{ + "name": meta["name"], + "namespace": meta["namespace"], + "labels": labels, + }, + } + return m, nil + } + } +} + +// function to get the yaml output of an object +func GetYamlOfObject(object interface{}) (string, error) { + if b, err := json.Marshal(object); err != nil { + return "", fmt.Errorf("Error while encoding %+v into JSON: %s", object, err) + } else { + var v map[string]interface{} + if err := json.Unmarshal(b, &v); err != nil { + return "", fmt.Errorf("Error while decoding JSON %s into an object: %s", v, err) + } else { + if y, err := yaml.Marshal(&v); err != nil { + return "", fmt.Errorf("Error while encoding Map %s into YAML: %s", v, err) + } else { + return string(y), nil + } + } + } +} + +// function to get yaml output of hash object for k8s objects +func GetYamlOfHashObjectOfK8sObject(object interface{}) (string, error) { + if hashObject, err := GetHashObjectOfKubernetes(object); err != nil { + return "", err + } else { + if yamlOutput, err := GetYamlOfObject(hashObject); err != nil { + return "", err + } else { + return yamlOutput, nil + } + } +} diff --git a/pkg/utils/helper_test.go b/pkg/utils/helper_test.go new file mode 100644 index 0000000..b4d321e --- /dev/null +++ b/pkg/utils/helper_test.go @@ -0,0 +1,173 @@ +package utils + +import ( + assert "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" + "encoding/json" +) + +func TestGetHashObjectOfKubernetes(t *testing.T) { + labels := map[string]string{ + "yelp.com/rick": "andmortyadventures", + "yelp.com/operator_config_hash": "somerandomhash", + } + labelsWithoutHash := map[string]string{ + "yelp.com/rick": "andmortyadventures", + } + replicas := int32(2) + someStatefulSet := &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "morty-test-cluster", + Namespace: "paasta-cassandra", + Labels: labels, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas, + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{corev1.PersistentVolumeClaim{}}, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{}, + Containers: []corev1.Container{}, + }, + }, + }, + } + expectedHashObject := map[string]interface{}{ + "kind": someStatefulSet.TypeMeta.Kind, + "apiVersion": someStatefulSet.TypeMeta.APIVersion, + "spec": someStatefulSet.Spec, + "metadata": map[string]interface{}{ + "name": someStatefulSet.ObjectMeta.Name, + "namespace": someStatefulSet.ObjectMeta.Namespace, + "labels": labelsWithoutHash, + }, + } + expectedOutString, err := json.Marshal(expectedHashObject) + _ = json.Unmarshal(expectedOutString, &expectedHashObject) + hashObject, err := GetHashObjectOfKubernetes(someStatefulSet) + if err != nil { + t.Errorf("Failed to calculate hash object") + } + assert.Equal(t, expectedHashObject, hashObject) +} + +func TestGetYamlOfObject(t *testing.T) { + labels := map[string]string{ + "yelp.com/rick": "andmortyadventures", + "yelp.com/operator_config_hash": "somerandomhash", + } + replicas := int32(2) + someStatefulSet := &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "morty-test-cluster", + Namespace: "paasta-cassandra", + Labels: labels, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas, + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{corev1.PersistentVolumeClaim{}}, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{}, + Containers: []corev1.Container{}, + }, + }, + }, + } + expectedOutput := `apiVersion: apps/v1 +kind: StatefulSet +metadata: + creationTimestamp: null + labels: + yelp.com/operator_config_hash: somerandomhash + yelp.com/rick: andmortyadventures + name: morty-test-cluster + namespace: paasta-cassandra +spec: + replicas: 2 + selector: null + serviceName: "" + template: + metadata: + creationTimestamp: null + spec: + containers: [] + updateStrategy: {} + volumeClaimTemplates: + - metadata: + creationTimestamp: null + spec: + resources: {} + status: {} +status: + replicas: 0 +` + actualOutput, _ := GetYamlOfObject(someStatefulSet) + assert.Equal(t, expectedOutput, actualOutput) +} + +func TestGetYamlOfHashObjectOfK8sObject(t *testing.T) { + labels := map[string]string{ + "yelp.com/rick": "andmortyadventures", + "yelp.com/operator_config_hash": "somerandomhash", + } + replicas := int32(2) + someStatefulSet := &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "morty-test-cluster", + Namespace: "paasta-cassandra", + Labels: labels, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas, + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{corev1.PersistentVolumeClaim{}}, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{}, + Containers: []corev1.Container{}, + }, + }, + }, + } + expectedOutput := `apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + yelp.com/rick: andmortyadventures + name: morty-test-cluster + namespace: paasta-cassandra +spec: + replicas: 2 + selector: null + serviceName: "" + template: + metadata: + creationTimestamp: null + spec: + containers: [] + updateStrategy: {} + volumeClaimTemplates: + - metadata: + creationTimestamp: null + spec: + resources: {} + status: {} +` + actualOutput, _ := GetYamlOfHashObjectOfK8sObject(someStatefulSet) + assert.Equal(t, expectedOutput, actualOutput) +} \ No newline at end of file From 78902a9e37898fd5f248ce802b943eba79f67777 Mon Sep 17 00:00:00 2001 From: Ashish Khatkar Date: Fri, 28 Feb 2020 17:02:42 +0000 Subject: [PATCH 2/4] Added logic to compute yaml diff, refactored hashing helper --- go.mod | 1 + pkg/hashing/hashing.go | 7 +- pkg/hashing/helper.go | 42 +++++++++ pkg/hashing/helper_test.go | 59 ++++++++++++ pkg/utils/helper.go | 74 --------------- pkg/utils/helper_test.go | 173 ----------------------------------- pkg/utils/yamlhelper.go | 55 +++++++++++ pkg/utils/yamlhelper_test.go | 89 ++++++++++++++++++ 8 files changed, 249 insertions(+), 251 deletions(-) create mode 100644 pkg/hashing/helper.go create mode 100644 pkg/hashing/helper_test.go delete mode 100644 pkg/utils/helper.go delete mode 100644 pkg/utils/helper_test.go create mode 100644 pkg/utils/yamlhelper.go create mode 100644 pkg/utils/yamlhelper_test.go diff --git a/go.mod b/go.mod index 17b08e8..4d18303 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/pborman/uuid v1.2.0 // indirect github.com/pkg/errors v0.8.1 + github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_golang v1.3.0 // indirect github.com/spf13/pflag v1.0.3 // indirect github.com/stretchr/testify v1.4.0 diff --git a/pkg/hashing/hashing.go b/pkg/hashing/hashing.go index 578d094..8264294 100644 --- a/pkg/hashing/hashing.go +++ b/pkg/hashing/hashing.go @@ -3,20 +3,19 @@ package hashing import ( "encoding/json" "fmt" - utils "github.com/Yelp/paasta-tools-go/pkg/utils" "hash/fnv" "k8s.io/apimachinery/pkg/util/rand" "reflect" ) func ComputeHashForKubernetesObject(object interface{}) (string, error) { - if m, err := utils.GetHashObjectOfKubernetes(object); err != nil { + if m, err := GetFilteredK8sObjectForHashing(object); err != nil { return "", err } else { // By using serialized JSON for hashing we're making the hashing process // a bit easier (like having maps always being sorted by keys). if b, err := json.Marshal(m); err != nil { - return "", fmt.Errorf("Error while encoding %+v into JSON: %s", m, err) + return "", fmt.Errorf("error while encoding %+v into JSON: %s", m, err) } else { hasher := fnv.New32a() hasher.Write(b) @@ -32,7 +31,7 @@ func SetKubernetesObjectHash(configHash string, object interface{}) error { if value.Kind() == reflect.Ptr { objectMeta = value.Elem().FieldByName("ObjectMeta") } else { - return fmt.Errorf("Must pass pointer to AddLabelsToMetadata so we can update labels using reflection.") + return fmt.Errorf("must pass pointer to AddLabelsToMetadata so we can update labels using reflection") } if objectMeta.Kind() == reflect.Struct { labels := objectMeta.FieldByName("Labels") diff --git a/pkg/hashing/helper.go b/pkg/hashing/helper.go new file mode 100644 index 0000000..4e66dd2 --- /dev/null +++ b/pkg/hashing/helper.go @@ -0,0 +1,42 @@ +package hashing + +import ( + "encoding/json" + "fmt" +) + +// function to get the relevant information out of k8s object that is needed to compute hash +func GetFilteredK8sObjectForHashing(object interface{}) (map[string]interface{}, error) { + // By marshaling/unmarshaling the object via JSON we're copying it into a + // map, which is easier to manipulate in generic way than structs. + if b, err := json.Marshal(object); err != nil { + return nil, fmt.Errorf("error while encoding %+v into JSON: %s", object, err) + } else { + var v map[string]interface{} + if err := json.Unmarshal(b, &v); err != nil { + return nil, fmt.Errorf("error while decoding JSON %s into an object: %s", v, err) + } else { + // We need only kind/version/spec and labels excluding the label with the + // current hash value while calculating the hash. Also Kubernetes adds + // its own info into `metadata` which we need to ignore. + meta := v["metadata"].(map[string]interface{}) + labels := meta["labels"] + if labels != nil { + delete(labels.(map[string]interface{}), "yelp.com/operator_config_hash") + } else { + labels = make(map[string]interface{}) + } + m := map[string]interface{}{ + "kind": v["kind"], + "apiVersion": v["apiVersion"], + "spec": v["spec"], + "metadata": map[string]interface{}{ + "name": meta["name"], + "namespace": meta["namespace"], + "labels": labels, + }, + } + return m, nil + } + } +} diff --git a/pkg/hashing/helper_test.go b/pkg/hashing/helper_test.go new file mode 100644 index 0000000..ed8aec5 --- /dev/null +++ b/pkg/hashing/helper_test.go @@ -0,0 +1,59 @@ +package hashing + +import ( + "encoding/json" + assert "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" +) + +func TestGetHashObjectOfKubernetes(t *testing.T) { + labels := map[string]string{ + "yelp.com/rick": "andmortyadventures", + "yelp.com/operator_config_hash": "somerandomhash", + } + labelsWithoutHash := map[string]string{ + "yelp.com/rick": "andmortyadventures", + } + replicas := int32(2) + someStatefulSet := &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "morty-test-cluster", + Namespace: "paasta-cassandra", + Labels: labels, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas, + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{corev1.PersistentVolumeClaim{}}, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{}, + Containers: []corev1.Container{}, + }, + }, + }, + } + expectedHashObject := map[string]interface{}{ + "kind": someStatefulSet.TypeMeta.Kind, + "apiVersion": someStatefulSet.TypeMeta.APIVersion, + "spec": someStatefulSet.Spec, + "metadata": map[string]interface{}{ + "name": someStatefulSet.ObjectMeta.Name, + "namespace": someStatefulSet.ObjectMeta.Namespace, + "labels": labelsWithoutHash, + }, + } + expectedOutString, err := json.Marshal(expectedHashObject) + _ = json.Unmarshal(expectedOutString, &expectedHashObject) + hashObject, err := GetFilteredK8sObjectForHashing(someStatefulSet) + if err != nil { + t.Errorf("Failed to calculate hash object") + } + assert.Equal(t, expectedHashObject, hashObject) +} diff --git a/pkg/utils/helper.go b/pkg/utils/helper.go deleted file mode 100644 index 1999c6d..0000000 --- a/pkg/utils/helper.go +++ /dev/null @@ -1,74 +0,0 @@ -package utils - -import ( - "encoding/json" - "fmt" - "gopkg.in/yaml.v2" -) - -// function to get the relevant information out of k8s object that is needed to compute hash -func GetHashObjectOfKubernetes(object interface{}) (map[string]interface{}, error) { - // By marshaling/unmarshaling the object via JSON we're copying it into a - // map, which is easier to manipulate in generic way than structs. - if b, err := json.Marshal(object); err != nil { - return nil, fmt.Errorf("Error while encoding %+v into JSON: %s", object, err) - } else { - var v map[string]interface{} - if err := json.Unmarshal(b, &v); err != nil { - return nil, fmt.Errorf("Error while decoding JSON %s into an object: %s", v, err) - } else { - // We need only kind/version/spec and labels excluding the label with the - // current hash value while calculating the hash. Also Kubernetes adds - // its own info into `metadata` which we need to ignore. - meta := v["metadata"].(map[string]interface{}) - labels := meta["labels"] - if labels != nil { - delete(labels.(map[string]interface{}), "yelp.com/operator_config_hash") - } else { - labels = make(map[string]interface{}) - } - m := map[string]interface{}{ - "kind": v["kind"], - "apiVersion": v["apiVersion"], - "spec": v["spec"], - "metadata": map[string]interface{}{ - "name": meta["name"], - "namespace": meta["namespace"], - "labels": labels, - }, - } - return m, nil - } - } -} - -// function to get the yaml output of an object -func GetYamlOfObject(object interface{}) (string, error) { - if b, err := json.Marshal(object); err != nil { - return "", fmt.Errorf("Error while encoding %+v into JSON: %s", object, err) - } else { - var v map[string]interface{} - if err := json.Unmarshal(b, &v); err != nil { - return "", fmt.Errorf("Error while decoding JSON %s into an object: %s", v, err) - } else { - if y, err := yaml.Marshal(&v); err != nil { - return "", fmt.Errorf("Error while encoding Map %s into YAML: %s", v, err) - } else { - return string(y), nil - } - } - } -} - -// function to get yaml output of hash object for k8s objects -func GetYamlOfHashObjectOfK8sObject(object interface{}) (string, error) { - if hashObject, err := GetHashObjectOfKubernetes(object); err != nil { - return "", err - } else { - if yamlOutput, err := GetYamlOfObject(hashObject); err != nil { - return "", err - } else { - return yamlOutput, nil - } - } -} diff --git a/pkg/utils/helper_test.go b/pkg/utils/helper_test.go deleted file mode 100644 index b4d321e..0000000 --- a/pkg/utils/helper_test.go +++ /dev/null @@ -1,173 +0,0 @@ -package utils - -import ( - assert "github.com/stretchr/testify/assert" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "testing" - "encoding/json" -) - -func TestGetHashObjectOfKubernetes(t *testing.T) { - labels := map[string]string{ - "yelp.com/rick": "andmortyadventures", - "yelp.com/operator_config_hash": "somerandomhash", - } - labelsWithoutHash := map[string]string{ - "yelp.com/rick": "andmortyadventures", - } - replicas := int32(2) - someStatefulSet := &appsv1.StatefulSet{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "StatefulSet", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "morty-test-cluster", - Namespace: "paasta-cassandra", - Labels: labels, - }, - Spec: appsv1.StatefulSetSpec{ - Replicas: &replicas, - VolumeClaimTemplates: []corev1.PersistentVolumeClaim{corev1.PersistentVolumeClaim{}}, - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Volumes: []corev1.Volume{}, - Containers: []corev1.Container{}, - }, - }, - }, - } - expectedHashObject := map[string]interface{}{ - "kind": someStatefulSet.TypeMeta.Kind, - "apiVersion": someStatefulSet.TypeMeta.APIVersion, - "spec": someStatefulSet.Spec, - "metadata": map[string]interface{}{ - "name": someStatefulSet.ObjectMeta.Name, - "namespace": someStatefulSet.ObjectMeta.Namespace, - "labels": labelsWithoutHash, - }, - } - expectedOutString, err := json.Marshal(expectedHashObject) - _ = json.Unmarshal(expectedOutString, &expectedHashObject) - hashObject, err := GetHashObjectOfKubernetes(someStatefulSet) - if err != nil { - t.Errorf("Failed to calculate hash object") - } - assert.Equal(t, expectedHashObject, hashObject) -} - -func TestGetYamlOfObject(t *testing.T) { - labels := map[string]string{ - "yelp.com/rick": "andmortyadventures", - "yelp.com/operator_config_hash": "somerandomhash", - } - replicas := int32(2) - someStatefulSet := &appsv1.StatefulSet{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "StatefulSet", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "morty-test-cluster", - Namespace: "paasta-cassandra", - Labels: labels, - }, - Spec: appsv1.StatefulSetSpec{ - Replicas: &replicas, - VolumeClaimTemplates: []corev1.PersistentVolumeClaim{corev1.PersistentVolumeClaim{}}, - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Volumes: []corev1.Volume{}, - Containers: []corev1.Container{}, - }, - }, - }, - } - expectedOutput := `apiVersion: apps/v1 -kind: StatefulSet -metadata: - creationTimestamp: null - labels: - yelp.com/operator_config_hash: somerandomhash - yelp.com/rick: andmortyadventures - name: morty-test-cluster - namespace: paasta-cassandra -spec: - replicas: 2 - selector: null - serviceName: "" - template: - metadata: - creationTimestamp: null - spec: - containers: [] - updateStrategy: {} - volumeClaimTemplates: - - metadata: - creationTimestamp: null - spec: - resources: {} - status: {} -status: - replicas: 0 -` - actualOutput, _ := GetYamlOfObject(someStatefulSet) - assert.Equal(t, expectedOutput, actualOutput) -} - -func TestGetYamlOfHashObjectOfK8sObject(t *testing.T) { - labels := map[string]string{ - "yelp.com/rick": "andmortyadventures", - "yelp.com/operator_config_hash": "somerandomhash", - } - replicas := int32(2) - someStatefulSet := &appsv1.StatefulSet{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "StatefulSet", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "morty-test-cluster", - Namespace: "paasta-cassandra", - Labels: labels, - }, - Spec: appsv1.StatefulSetSpec{ - Replicas: &replicas, - VolumeClaimTemplates: []corev1.PersistentVolumeClaim{corev1.PersistentVolumeClaim{}}, - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Volumes: []corev1.Volume{}, - Containers: []corev1.Container{}, - }, - }, - }, - } - expectedOutput := `apiVersion: apps/v1 -kind: StatefulSet -metadata: - labels: - yelp.com/rick: andmortyadventures - name: morty-test-cluster - namespace: paasta-cassandra -spec: - replicas: 2 - selector: null - serviceName: "" - template: - metadata: - creationTimestamp: null - spec: - containers: [] - updateStrategy: {} - volumeClaimTemplates: - - metadata: - creationTimestamp: null - spec: - resources: {} - status: {} -` - actualOutput, _ := GetYamlOfHashObjectOfK8sObject(someStatefulSet) - assert.Equal(t, expectedOutput, actualOutput) -} \ No newline at end of file diff --git a/pkg/utils/yamlhelper.go b/pkg/utils/yamlhelper.go new file mode 100644 index 0000000..33c3d99 --- /dev/null +++ b/pkg/utils/yamlhelper.go @@ -0,0 +1,55 @@ +package utils + +import ( + "encoding/json" + "fmt" + "github.com/pmezard/go-difflib/difflib" + "gopkg.in/yaml.v2" +) + +// function to get the yaml output of an object +// marshal/unmarshal to and from json in order to maintain consistency between json and yaml +// currently k8s objects have json field tags, so this helps in skipping empty fields +// and give consistent output yaml similar to json +func getYamlOfObject(object interface{}) (string, error) { + if b, err := json.Marshal(object); err != nil { + return "", fmt.Errorf("error while encoding %+v into JSON: %s", object, err) + } else { + var v map[string]interface{} + if err := json.Unmarshal(b, &v); err != nil { + return "", fmt.Errorf("error while decoding JSON %s into an object: %s", v, err) + } else { + if y, err := yaml.Marshal(&v); err != nil { + return "", fmt.Errorf("error while encoding Map %s into YAML: %s", v, err) + } else { + return string(y), nil + } + } + } +} + +func generateYamlDiff(yaml1 string, yaml2 string) string { + diff, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ + A: difflib.SplitLines(yaml1), + B: difflib.SplitLines(yaml2), + FromFile: "Old", + FromDate: "", + ToFile: "New", + ToDate: "", + Context: 1, + }) + return diff +} + +// func to generate yaml diff of objects used for hashing +func GetYamlDiffForObjects(objectOld interface{}, objectNew interface{}) (string, error) { + yamlOld, err := getYamlOfObject(objectOld) + if err != nil { + return "", err + } + yamlNew, err := getYamlOfObject(objectNew) + if err != nil { + return "", err + } + return generateYamlDiff(yamlOld, yamlNew), nil +} diff --git a/pkg/utils/yamlhelper_test.go b/pkg/utils/yamlhelper_test.go new file mode 100644 index 0000000..148eb0e --- /dev/null +++ b/pkg/utils/yamlhelper_test.go @@ -0,0 +1,89 @@ +package utils + +import ( + assert "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" +) + +func TestGetYamlDiffForObjects(t *testing.T) { + labels1 := map[string]string{ + "yelp.com/rick": "andmortyadventures1", + "yelp.com/operator_config_hash": "somerandomhash1", + } + replicas1 := int32(2) + someStatefulSet1 := &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "morty-test-cluster1", + Namespace: "paasta-cassandra", + Labels: labels1, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas1, + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{corev1.PersistentVolumeClaim{}}, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{}, + Containers: []corev1.Container{}, + }, + }, + }, + } + + labels2 := map[string]string{ + "yelp.com/rick": "andmortyadventures2", + "yelp.com/operator_config_hash": "somerandomhash2", + } + replicas2 := int32(2) + someStatefulSet2 := &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "morty-test-cluster2", + Namespace: "paasta-cassandra", + Labels: labels2, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas2, + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{corev1.PersistentVolumeClaim{}}, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: "volume1", + VolumeSource: corev1.VolumeSource{}, + }, + }, + Containers: []corev1.Container{}, + }, + }, + }, + } + expectedDiff := `--- Old ++++ New +@@ -5,5 +5,5 @@ + labels: +- yelp.com/operator_config_hash: somerandomhash1 +- yelp.com/rick: andmortyadventures1 +- name: morty-test-cluster1 ++ yelp.com/operator_config_hash: somerandomhash2 ++ yelp.com/rick: andmortyadventures2 ++ name: morty-test-cluster2 + namespace: paasta-cassandra +@@ -18,2 +18,4 @@ + containers: [] ++ volumes: ++ - name: volume1 + updateStrategy: {} +` + actualDiff, _ := GetYamlDiffForObjects(someStatefulSet1, someStatefulSet2) + assert.Equal(t, expectedDiff, actualDiff) +} From 7b7bb7c1691a05ee972bf7a0831b0282d4e96d29 Mon Sep 17 00:00:00 2001 From: Ashish Khatkar Date: Mon, 2 Mar 2020 11:43:52 +0000 Subject: [PATCH 3/4] Nit changes for review --- pkg/utils/yamlhelper.go | 10 ++++++---- pkg/utils/yamlhelper_test.go | 14 +++++++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/pkg/utils/yamlhelper.go b/pkg/utils/yamlhelper.go index 33c3d99..7c1d280 100644 --- a/pkg/utils/yamlhelper.go +++ b/pkg/utils/yamlhelper.go @@ -28,7 +28,7 @@ func getYamlOfObject(object interface{}) (string, error) { } } -func generateYamlDiff(yaml1 string, yaml2 string) string { +func generateYamlDiff(yaml1 string, yaml2 string, context int) string { diff, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ A: difflib.SplitLines(yaml1), B: difflib.SplitLines(yaml2), @@ -36,13 +36,15 @@ func generateYamlDiff(yaml1 string, yaml2 string) string { FromDate: "", ToFile: "New", ToDate: "", - Context: 1, + Context: context, }) return diff } // func to generate yaml diff of objects used for hashing -func GetYamlDiffForObjects(objectOld interface{}, objectNew interface{}) (string, error) { +// context : number of context lines to use for generating diff +// for more reference on context : https://github.com/pmezard/go-difflib/blob/5d4384ee4fb2527b0a1256a821ebfc92f91efefc/difflib/difflib.go#L559 +func GetYamlDiffForObjects(objectOld interface{}, objectNew interface{}, context int) (string, error) { yamlOld, err := getYamlOfObject(objectOld) if err != nil { return "", err @@ -51,5 +53,5 @@ func GetYamlDiffForObjects(objectOld interface{}, objectNew interface{}) (string if err != nil { return "", err } - return generateYamlDiff(yamlOld, yamlNew), nil + return generateYamlDiff(yamlOld, yamlNew, context), nil } diff --git a/pkg/utils/yamlhelper_test.go b/pkg/utils/yamlhelper_test.go index 148eb0e..52dd910 100644 --- a/pkg/utils/yamlhelper_test.go +++ b/pkg/utils/yamlhelper_test.go @@ -69,7 +69,9 @@ func TestGetYamlDiffForObjects(t *testing.T) { } expectedDiff := `--- Old +++ New -@@ -5,5 +5,5 @@ +@@ -3,9 +3,9 @@ + metadata: + creationTimestamp: null labels: - yelp.com/operator_config_hash: somerandomhash1 - yelp.com/rick: andmortyadventures1 @@ -78,12 +80,18 @@ func TestGetYamlDiffForObjects(t *testing.T) { + yelp.com/rick: andmortyadventures2 + name: morty-test-cluster2 namespace: paasta-cassandra -@@ -18,2 +18,4 @@ + spec: + replicas: 2 +@@ -16,6 +16,8 @@ + creationTimestamp: null + spec: containers: [] + volumes: + - name: volume1 updateStrategy: {} + volumeClaimTemplates: + - metadata: ` - actualDiff, _ := GetYamlDiffForObjects(someStatefulSet1, someStatefulSet2) + actualDiff, _ := GetYamlDiffForObjects(someStatefulSet1, someStatefulSet2, 3) assert.Equal(t, expectedDiff, actualDiff) } From 9afe192f00fba1aa1d3243370fe6773d6fc7948f Mon Sep 17 00:00:00 2001 From: Ashish Khatkar Date: Mon, 2 Mar 2020 12:12:15 +0000 Subject: [PATCH 4/4] Updating link for unified context --- pkg/utils/yamlhelper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/utils/yamlhelper.go b/pkg/utils/yamlhelper.go index 7c1d280..7bca0d8 100644 --- a/pkg/utils/yamlhelper.go +++ b/pkg/utils/yamlhelper.go @@ -43,7 +43,7 @@ func generateYamlDiff(yaml1 string, yaml2 string, context int) string { // func to generate yaml diff of objects used for hashing // context : number of context lines to use for generating diff -// for more reference on context : https://github.com/pmezard/go-difflib/blob/5d4384ee4fb2527b0a1256a821ebfc92f91efefc/difflib/difflib.go#L559 +// for more reference on context : https://www.gnu.org/software/diffutils/manual/html_node/Unified-Format.html#Unified-Format func GetYamlDiffForObjects(objectOld interface{}, objectNew interface{}, context int) (string, error) { yamlOld, err := getYamlOfObject(objectOld) if err != nil {