Merge pull request #324 from MUzairS15/meshmodel-generation

[Meshmodel] Add support for x-kubernetes-preserve-unknown-fields

go.mod

@@ -12,7 +12,7 @@ replace (
 )
 
 require (
-	cuelang.org/go v0.4.3
+	cuelang.org/go v0.5.0
 	github.com/compose-spec/compose-go v1.0.8
 	github.com/docker/compose/v2 v2.2.0
 	github.com/go-git/go-git/v5 v5.4.2
@@ -99,7 +99,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
-	github.com/cockroachdb/apd/v2 v2.0.1 // indirect
+	github.com/cockroachdb/apd/v2 v2.0.2 // indirect
 	github.com/containerd/cgroups v1.0.4 // indirect
 	github.com/containerd/containerd v1.6.19 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
@@ -177,7 +177,7 @@ require (
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/mitchellh/go-wordwrap v1.0.0 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/moby/locker v1.0.1 // indirect

go.sum

@@ -40,8 +40,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
-cuelang.org/go v0.4.3 h1:W3oBBjDTm7+IZfCKZAmC8uDG0eYfJL4Pp/xbbCMKaVo=
-cuelang.org/go v0.4.3/go.mod h1:7805vR9H+VoBNdWFdI7jyDR3QLUPp4+naHfbcgp55HI=
+cuelang.org/go v0.5.0 h1:D6N0UgTGJCOxFKU8RU+qYvavKNsVc/+ZobmifStVJzU=
+cuelang.org/go v0.5.0/go.mod h1:okjJBHFQFer+a41sAe2SaGm1glWS8oEb6CmJvn5Zdws=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/AlecAivazis/survey/v2 v2.3.6 h1:NvTuVHISgTHEHeBFqt6BHOe4Ny/NwGZr7w+F8S9ziyw=
 github.com/AlecAivazis/survey/v2 v2.3.6/go.mod h1:4AuI9b7RjAR+G7v9+C4YSlX/YL3K3cWNXgWXOhllqvI=
@@ -212,8 +212,8 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht
 github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
-github.com/cockroachdb/apd/v2 v2.0.1 h1:y1Rh3tEU89D+7Tgbw+lp52T6p/GJLpDmNvr10UWqLTE=
-github.com/cockroachdb/apd/v2 v2.0.1/go.mod h1:DDxRlzC2lo3/vSlmSoS7JkqbbrARPuFOGr0B9pvN3Gw=
+github.com/cockroachdb/apd/v2 v2.0.2 h1:weh8u7Cneje73dDh+2tEVLUvyBc89iwepWCD8b8034E=
+github.com/cockroachdb/apd/v2 v2.0.2/go.mod h1:DDxRlzC2lo3/vSlmSoS7JkqbbrARPuFOGr0B9pvN3Gw=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/compose-spec/compose-go v1.0.8 h1:fgT7mYYu5Sp37i2lUIAAvwJpkAHk6dP5ITHy/LlutUk=
@@ -431,7 +431,7 @@ github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE=
 github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
-github.com/emicklei/proto v1.6.15 h1:XbpwxmuOPrdES97FrSfpyy67SSCV/wBIKXqgJzh6hNw=
+github.com/emicklei/proto v1.10.0 h1:pDGyFRVV5RvV+nkBK9iy3q67FBy9Xa7vwrOTE+g5aGw=
 github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
 github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -961,8 +961,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
-github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4=
-github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
+github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
+github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
 github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ=
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
@@ -1167,7 +1167,7 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
 github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/protocolbuffers/txtpbfmt v0.0.0-20201118171849-f6a6b3f636fc h1:gSVONBi2HWMFXCa9jFdYvYk7IwW/mTLxWOF7rXS4LO0=
+github.com/protocolbuffers/txtpbfmt v0.0.0-20220428173112-74888fd59c2b h1:zd/2RNzIRkoGGMjE+YIsZ85CnDIz672JK2F3Zl4vux4=
 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=
 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=

utils/component/error.go

@@ -3,19 +3,26 @@ package component
 import "github.com/layer5io/meshkit/errors"
 
 const (
-	ErrCrdGenerateCode = "11088"
-	ErrDefinitionCode  = "11090"
-	ErrGetSchemaCode   = "11091"
+	ErrCrdGenerateCode  = "11088"
+	ErrDefinitionCode   = "11090"
+	ErrGetSchemaCode    = "11091"
+	ErrUpdateSchemaCode = "11092"
 )
 
+// No reference usage found. Also check in adapters before deleting
 func ErrCrdGenerate(err error) error {
-	return errors.New(ErrCrdGenerateCode, errors.Alert, []string{"Could not generate component with the given CRD"}, []string{err.Error()}, []string{""}, []string{"Make sure that the crds provided are all valid YAML"})
+	return errors.New(ErrCrdGenerateCode, errors.Alert, []string{"Could not generate component with the given CRD"}, []string{err.Error()}, []string{""}, []string{"Verify CRD has valid schema."})
 }
 
+// No reference usage found. Also check in adapters before deleting
 func ErrGetDefinition(err error) error {
-	return errors.New(ErrDefinitionCode, errors.Alert, []string{"Could not get definition for the given CRD"}, []string{err.Error()}, []string{""}, []string{"Make sure that the given CRD is valid"})
+	return errors.New(ErrDefinitionCode, errors.Alert, []string{"Could not get definition for the given CRD"}, []string{err.Error()}, []string{""}, []string{"Verify CRD has valid schema."})
 }
 
 func ErrGetSchema(err error) error {
-	return errors.New(ErrGetSchemaCode, errors.Alert, []string{"Could not get schema for the given CRD"}, []string{err.Error()}, []string{""}, []string{"Make sure that the given CRD is valid"})
+	return errors.New(ErrGetSchemaCode, errors.Alert, []string{"Could not get schema for the given CRD"}, []string{err.Error()}, []string{"Unable to marshal from cue value to JSON", "Unable to unmarshal from JSON to Go type"}, []string{"Verify CRD has valid schema.", "Malformed JSON provided", "CUE path to propery doesn't exist"})
+}
+
+func ErrUpdateSchema(err error, obj string) error {
+	return errors.New(ErrUpdateSchemaCode, errors.Alert, []string{"Failed to update schema properties for ", obj}, []string{err.Error()}, []string{"Incorrect type assertion", "Selector.Unquoted might have been invoked on non-string label", "error during conversion from cue.Selector to string"}, []string{"Ensure correct type assertion", "Perform appropriate conversion from cue.Selector to string", "Verify CRD has valid schema"})
 }

utils/component/generator.go

@@ -1,8 +1,11 @@
 package component
 
 import (
+	"encoding/json"
+	"errors"
 	"fmt"
 
+	"cuelang.org/go/cue"
 	"github.com/layer5io/meshkit/models/meshmodel/core/v1alpha1"
 	"github.com/layer5io/meshkit/utils"
 	"github.com/layer5io/meshkit/utils/manifests"
@@ -12,11 +15,12 @@ const ComponentMetaNameKey = "name"
 
 // all paths should be a valid CUE expression
 type CuePathConfig struct {
-	NamePath    string
-	GroupPath   string
-	VersionPath string
-	SpecPath    string
-	ScopePath   string
+	NamePath       string
+	GroupPath      string
+	VersionPath    string
+	SpecPath       string
+	ScopePath      string
+	PropertiesPath string
 	// identifiers are the values that uniquely identify a CRD (in most of the cases, it is the 'Name' field)
 	IdentifierPath string
 }
@@ -28,6 +32,7 @@ var DefaultPathConfig = CuePathConfig{
 	GroupPath:      "spec.group",
 	ScopePath:      "spec.scope",
 	SpecPath:       "spec.versions[0].schema.openAPIV3Schema.properties.spec",
+	PropertiesPath: "properties",
 }
 
 var DefaultPathConfig2 = CuePathConfig{
@@ -86,3 +91,85 @@ func Generate(crd string) (v1alpha1.ComponentDefinition, error) {
 	component.DisplayName = manifests.FormatToReadableString(name)
 	return component, nil
 }
+
+/*
+Find and modify specific schema properties.
+1. Identify interesting properties by walking entire schema.
+2. Store path to interesting properties. Finish walk.
+3. Iterate all paths and modify properties.
+5. If error occurs, return nil and skip modifications.
+*/
+func UpdateProperties(fieldVal cue.Value, cuePath cue.Path, group string) (map[string]interface{}, error) {
+	rootPath := fieldVal.Path().Selectors()
+
+	compProperties := fieldVal.LookupPath(cuePath)
+	crd, err := fieldVal.MarshalJSON()
+	if err != nil {
+		return nil, ErrUpdateSchema(err, group)
+	}
+
+	modified := make(map[string]interface{})
+	pathSelectors := [][]cue.Selector{}
+
+	err = json.Unmarshal(crd, &modified)
+	if err != nil {
+		return nil, ErrUpdateSchema(err, group)
+	}
+
+	compProperties.Walk(func(c cue.Value) bool {
+		return true
+	}, func(c cue.Value) {
+		val := c.LookupPath(cue.ParsePath(`"x-kubernetes-preserve-unknown-fields"`))
+		if val.Exists() {
+			child := val.Path().Selectors()
+			childM := child[len(rootPath):(len(child) - 1)]
+			pathSelectors = append(pathSelectors, childM)
+		}
+	})
+
+	// "pathSelectors" contains all the paths from root to the property which needs to be modified.
+	for _, selectors := range pathSelectors {
+		var m interface{}
+		m = modified
+		index := 0
+
+		for index < len(selectors) {
+			selector := selectors[index]
+			selectorType := selector.Type()
+			s := selector.String()
+			if selectorType == cue.IndexLabel {
+				t, ok := m.([]interface{})
+				if !ok {
+					return nil, ErrUpdateSchema(errors.New("error converting to []interface{}"), group)
+				}
+				token := selector.Index()
+				m, ok = t[token].(map[string]interface{})
+				if !ok {
+					return nil, ErrUpdateSchema(errors.New("error converting to map[string]interface{}"), group)
+				}
+			} else {
+				if selectorType == cue.StringLabel {
+					s = selector.Unquoted()
+				}
+				t, ok := m.(map[string]interface{})
+				if !ok {
+					return nil, ErrUpdateSchema(errors.New("error converting to map[string]interface{}"), group)
+				}
+				m = t[s]
+			}
+			index++
+		}
+
+		t, ok := m.(map[string]interface{})
+		if !ok {
+			return nil, ErrUpdateSchema(errors.New("error converting to map[string]interface{}"), group)
+		}
+		delete(t, "x-kubernetes-preserve-unknown-fields")
+		if m == nil {
+			m = make(map[string]interface{})
+		}
+		t["type"] = "string"
+		t["format"] = "textarea"
+	}
+	return modified, nil
+}

utils/component/utils.go

@@ -27,6 +27,15 @@ func getSchema(parsedCrd cue.Value, pathConf CuePathConfig) (string, error) {
 	if err != nil {
 		return "", ErrGetSchema(err)
 	}
+
+	updatedProps, err := UpdateProperties(specCueVal, cue.ParsePath(pathConf.PropertiesPath), resourceId)
+
+	if err != nil {
+		return "", err
+	}
+
+	schema = updatedProps
+
 	(schema)["title"] = manifests.FormatToReadableString(resourceId)
 	var output []byte
 	output, err = json.MarshalIndent(schema, "", " ")