Skip to content

Commit

Permalink
Merge pull request #603 from meshery/revert-600-revert-582-MUzairS15/…
Browse files Browse the repository at this point in the history
…generator/openapi

Revert "Revert "support generation of model from openapi schemas""
  • Loading branch information
Mohd Uzair authored Oct 3, 2024
2 parents 38beeb7 + 791faef commit 3addd46
Show file tree
Hide file tree
Showing 11 changed files with 282 additions and 45 deletions.
15 changes: 15 additions & 0 deletions encoding/convert.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package encoding

import (
"gopkg.in/yaml.v3"
)

func ToYaml(data []byte) ([]byte, error) {
var out map[string]interface{}
err := Unmarshal(data, &out)
if err != nil {
return nil, err
}

return yaml.Marshal(out)
}
8 changes: 8 additions & 0 deletions generators/artifacthub/package.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,14 @@ func (pkg AhPackage) GetVersion() string {
return pkg.Version
}

func (pkg AhPackage) GetSourceURL() string {
return pkg.ChartUrl
}

func (pkg AhPackage) GetName() string {
return pkg.Name
}

func (pkg AhPackage) GenerateComponents() ([]_component.ComponentDefinition, error) {
components := make([]_component.ComponentDefinition, 0)
// TODO: Move this to the configuration
Expand Down
59 changes: 40 additions & 19 deletions generators/github/package.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

"github.com/layer5io/meshkit/utils"
"github.com/layer5io/meshkit/utils/component"
"github.com/layer5io/meshkit/utils/kubernetes"
"github.com/layer5io/meshkit/utils/manifests"
"github.com/meshery/schemas/models/v1beta1/category"
_component "github.com/meshery/schemas/models/v1beta1/component"
Expand All @@ -25,6 +26,14 @@ func (gp GitHubPackage) GetVersion() string {
return gp.version
}

func (gp GitHubPackage) GetSourceURL() string {
return gp.SourceURL
}

func (gp GitHubPackage) GetName() string {
return gp.Name
}

func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, error) {
components := make([]_component.ComponentDefinition, 0)

Expand All @@ -34,28 +43,40 @@ func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition,
}

manifestBytes := bytes.Split(data, []byte("\n---\n"))
crds, errs := component.FilterCRDs(manifestBytes)
errs := []error{}

for _, crd := range crds {
comp, err := component.Generate(crd)
if err != nil {
continue
}
if comp.Model.Metadata == nil {
comp.Model.Metadata = &model.ModelDefinition_Metadata{}
}
if comp.Model.Metadata.AdditionalProperties == nil {
comp.Model.Metadata.AdditionalProperties = make(map[string]interface{})
}
for _, crd := range manifestBytes {
isCrd := kubernetes.IsCRD(string(crd))
if !isCrd {

comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL
comp.Model.Version = gp.version
comp.Model.Name = gp.Name
comp.Model.Category = category.CategoryDefinition{
Name: "",
comps, err := component.GenerateFromOpenAPI(string(crd), gp)
if err != nil {
errs = append(errs, component.ErrGetSchema(err))
continue
}
components = append(components, comps...)
} else {
comp, err := component.Generate(string(crd))
if err != nil {
continue
}
if comp.Model.Metadata == nil {
comp.Model.Metadata = &model.ModelDefinition_Metadata{}
}
if comp.Model.Metadata.AdditionalProperties == nil {
comp.Model.Metadata.AdditionalProperties = make(map[string]interface{})
}

comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL
comp.Model.Version = gp.version
comp.Model.Name = gp.Name
comp.Model.Category = category.CategoryDefinition{
Name: "",
}
comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name)
components = append(components, comp)
}
comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name)
components = append(components, comp)

}

return components, utils.CombineErrors(errs, "\n")
Expand Down
2 changes: 2 additions & 0 deletions generators/models/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ type Validator interface {
type Package interface {
GenerateComponents() ([]component.ComponentDefinition, error)
GetVersion() string
GetSourceURL() string
GetName() string
}

// Supports pulling packages from Artifact Hub and other sources like Docker Hub.
Expand Down
2 changes: 2 additions & 0 deletions utils/component/error.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ const (
ErrUpdateSchemaCode = "meshkit-11158"
)

var ErrNoSchemasFound = errors.New(ErrGetSchemaCode, errors.Alert, []string{"Could not get schema for the given openapi spec"}, []string{"The OpenAPI spec doesn't include \"components.schemas\" path"}, []string{"The spec doesn't have include any schema"}, []string{"Verify the spec has valid schema."})

// No reference usage found. Also check in adapters before deleting
func ErrCrdGenerate(err error) error {
return errors.New(ErrCrdGenerateCode, errors.Alert, []string{"Could not generate component with the given CRD"}, []string{err.Error()}, []string{""}, []string{"Verify CRD has valid schema."})
Expand Down
15 changes: 13 additions & 2 deletions utils/component/generator.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,17 +45,28 @@ var DefaultPathConfig2 = CuePathConfig{
SpecPath: "spec.validation.openAPIV3Schema",
}

var OpenAPISpecPathConfig = CuePathConfig{
NamePath: `x-kubernetes-group-version-kind"[0].kind`,
IdentifierPath: "spec.names.kind",
VersionPath: `"x-kubernetes-group-version-kind"[0].version`,
GroupPath: `"x-kubernetes-group-version-kind"[0].group`,
ScopePath: "spec.scope",
SpecPath: "spec.versions[0].schema.openAPIV3Schema",
PropertiesPath: "properties",
}

var Configs = []CuePathConfig{DefaultPathConfig, DefaultPathConfig2}

func Generate(crd string) (component.ComponentDefinition, error) {
func Generate(resource string) (component.ComponentDefinition, error) {
cmp := component.ComponentDefinition{}
cmp.SchemaVersion = v1beta1.ComponentSchemaVersion

cmp.Metadata = component.ComponentDefinition_Metadata{}
crdCue, err := utils.YamlToCue(crd)
crdCue, err := utils.YamlToCue(resource)
if err != nil {
return cmp, err
}

var schema string
for _, cfg := range Configs {
schema, err = getSchema(crdCue, cfg)
Expand Down
167 changes: 167 additions & 0 deletions utils/component/openapi_generator.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
package component

import (
"encoding/json"
"errors"
"fmt"
"strings"

"cuelang.org/go/cue"
"cuelang.org/go/cue/cuecontext"
cueJson "cuelang.org/go/encoding/json"
"github.com/layer5io/meshkit/generators/models"
"github.com/layer5io/meshkit/utils"
"github.com/layer5io/meshkit/utils/manifests"

"gopkg.in/yaml.v3"

"github.com/meshery/schemas/models/v1beta1"
"github.com/meshery/schemas/models/v1beta1/component"
"github.com/meshery/schemas/models/v1beta1/model"
)

func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.ComponentDefinition, error) {
if resource == "" {
return nil, nil
}
resource, err := getResolvedManifest(resource)
if err != nil && errors.Is(err, ErrNoSchemasFound) {
return nil, nil
}
if err != nil {
return nil, err
}
cuectx := cuecontext.New()
cueParsedManExpr, err := cueJson.Extract("", []byte(resource))
if err != nil {
return nil, err
}

parsedManifest := cuectx.BuildExpr(cueParsedManExpr)
definitions, err := utils.Lookup(parsedManifest, "components.schemas")

if err != nil {
return nil, err
}

fields, err := definitions.Fields()
if err != nil {
fmt.Printf("%v\n", err)
return nil, err
}
components := make([]component.ComponentDefinition, 0)

for fields.Next() {
fieldVal := fields.Value()
kindCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].kind`)
if err != nil {
continue
}
kind, err := kindCue.String()
kind = strings.ToLower(kind)
if err != nil {
fmt.Printf("%v", err)
continue
}

crd, err := fieldVal.MarshalJSON()
if err != nil {
fmt.Printf("%v", err)
continue
}
versionCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].version`)
if err != nil {
continue
}

groupCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].group`)
if err != nil {
continue
}

apiVersion, _ := versionCue.String()
if g, _ := groupCue.String(); g != "" {
apiVersion = g + "/" + apiVersion
}
modified := make(map[string]interface{}) //Remove the given fields which is either not required by End user (like status) or is prefilled by system (like apiVersion, kind and metadata)
err = json.Unmarshal(crd, &modified)
if err != nil {
fmt.Printf("%v", err)
continue
}

modifiedProps, err := UpdateProperties(fieldVal, cue.ParsePath("properties.spec"), apiVersion)
if err == nil {
modified = modifiedProps
}

DeleteFields(modified)
crd, err = json.Marshal(modified)
if err != nil {
fmt.Printf("%v", err)
continue
}

c := component.ComponentDefinition{
SchemaVersion: v1beta1.ComponentSchemaVersion,
Format: component.JSON,
Component: component.Component{
Kind: kind,
Version: apiVersion,
Schema: string(crd),
},
DisplayName: manifests.FormatToReadableString(kind),
Model: model.ModelDefinition{
SchemaVersion: v1beta1.ModelSchemaVersion,
Model: model.Model{
Version: pkg.GetVersion(),
},
Name: pkg.GetName(),
DisplayName: manifests.FormatToReadableString(pkg.GetName()),
Metadata: &model.ModelDefinition_Metadata{
AdditionalProperties: map[string]interface{}{
"source_uri": pkg.GetSourceURL(),
},
},
},
}

components = append(components, c)
}
return components, nil

}

func getResolvedManifest(manifest string) (string, error) {
var m map[string]interface{}

err := yaml.Unmarshal([]byte(manifest), &m)
if err != nil {
return "", utils.ErrDecodeYaml(err)
}

byt, err := json.Marshal(m)
if err != nil {
return "", utils.ErrMarshal(err)
}

cuectx := cuecontext.New()
cueParsedManExpr, err := cueJson.Extract("", byt)
if err != nil {
return "", ErrGetSchema(err)
}

parsedManifest := cuectx.BuildExpr(cueParsedManExpr)
definitions, err := utils.Lookup(parsedManifest, "components.schemas")
if err != nil {
return "", ErrNoSchemasFound
}
resol := manifests.ResolveOpenApiRefs{}
cache := make(map[string][]byte)
resolved, err := resol.ResolveReferences(byt, definitions, cache)
if err != nil {
return "", err
}
manifest = string(resolved)
return manifest, nil
}
16 changes: 3 additions & 13 deletions utils/component/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"github.com/layer5io/meshkit/utils"
"github.com/layer5io/meshkit/utils/kubernetes"
"github.com/layer5io/meshkit/utils/manifests"
"gopkg.in/yaml.v2"
)

// Remove the fields which is either not required by end user (like status) or is prefilled by system (like apiVersion, kind and metadata)
Expand Down Expand Up @@ -81,19 +80,10 @@ func FilterCRDs(manifests [][]byte) ([]string, []error) {
var errs []error
var filteredManifests []string
for _, m := range manifests {

var crd map[string]interface{}
err := yaml.Unmarshal(m, &crd)
if err != nil {
errs = append(errs, err)
continue
}

isCrd := kubernetes.IsCRD(crd)
if !isCrd {
continue
isCrd := kubernetes.IsCRD(string(m))
if isCrd {
filteredManifests = append(filteredManifests, string(m))
}
filteredManifests = append(filteredManifests, string(m))
}
return filteredManifests, errs
}
9 changes: 8 additions & 1 deletion utils/helm/helm.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"regexp"
"strings"

"github.com/layer5io/meshkit/encoding"
"github.com/layer5io/meshkit/utils"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
Expand Down Expand Up @@ -108,7 +109,13 @@ func writeToFile(w io.Writer, path string) error {
if err != nil {
return utils.ErrReadFile(err, path)
}
_, err = w.Write(data)

byt, err := encoding.ToYaml(data)
if err != nil {
return utils.ErrWriteFile(err, path)
}

_, err = w.Write(byt)
if err != nil {
return utils.ErrWriteFile(err, path)
}
Expand Down
Loading

0 comments on commit 3addd46

Please sign in to comment.