Skip to content

Commit

Permalink
Merge pull request #589 from Jougan-0/updateRegistrationLogic
Browse files Browse the repository at this point in the history 
update registration logic to support import command w/signoff
  • Loading branch information
@Jougan-0
Jougan-0 authored Sep 21, 2024
2 parents 535031d + 0072758 commit 1366959
Show file tree
Hide file tree
Showing 10 changed files with 488 additions and 83 deletions.
4 changes: 2 additions & 2 deletions models/meshmodel/registry/v1beta1/model_filter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e

if mf.Greedy {
if mf.Id != "" {
finder = finder.First("model_dbs.id = ?", mf.Id)
finder = finder.Where("model_dbs.id = ?", mf.Id)
}
if mf.Name != "" && mf.DisplayName != "" {
finder = finder.Where("model_dbs.name LIKE ? OR model_dbs.display_name LIKE ?", "%"+mf.Name+"%", "%"+mf.DisplayName+"%")
Expand Down Expand Up @@ -182,7 +182,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e
if includeComponents {
var components []component.ComponentDefinition
finder := db.Model(&component.ComponentDefinition{}).
Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version,component_definition_dbs.styles").
Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version,component_definition_dbs.styles,component_definition_dbs.capabilities").
Where("component_definition_dbs.model_id = ?", _modelDB.Id)
if err := finder.Scan(&components).Error; err != nil {
return nil, 0, 0, err
Expand Down
179 changes: 149 additions & 30 deletions models/registration/dir.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@ import (
"reflect"

"github.com/layer5io/meshkit/models/meshmodel/entity"
"github.com/layer5io/meshkit/models/oci"

"github.com/layer5io/meshkit/utils"

"github.com/meshery/schemas/models/v1alpha3/relationship"
"github.com/meshery/schemas/models/v1beta1/component"
"github.com/meshery/schemas/models/v1beta1/model"
Expand All @@ -26,69 +29,185 @@ func NewDir(path string) Dir {
}

/*
PkgUnit parses all the files inside the directory and finds out if they are any valid meshery definitions. Valid meshery definitions are added to the packagingUnit struct.
PkgUnit parses all the files inside the directory and finds out if they are any valid meshery definitions. Valid meshery definitions are added to the PackagingUnit struct.
Invalid definitions are stored in the regErrStore with error data.
*/
func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ packagingUnit, err error) {
pkg := packagingUnit{}
// check if the given is a directory
_, err = os.ReadDir(d.dirpath)
func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ PackagingUnit, err error) {
pkg := PackagingUnit{}

// Extract the filename to use as entityName in case of errors
filename := filepath.Base(d.dirpath)

// Check if the given path is accessible
_, err = os.Stat(d.dirpath)
if err != nil {
return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("Could not read the directory: %e", err))
regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filename, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not access the path: %w", err)))
return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not access the path: %w", err))
}

// Process the path (file or directory)
err = processDir(d.dirpath, &pkg, regErrStore)
if err != nil {
modelName := ""
if !reflect.ValueOf(pkg.Model).IsZero() {
modelName = pkg.Model.Name
}
regErrStore.InsertEntityRegError("", modelName, entity.EntityType("unknown"), filename, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not process the path: %w", err)))
return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not process the path: %w", err))
}
err = filepath.Walk(d.dirpath, func(path string, f os.FileInfo, err error) error {

if reflect.ValueOf(pkg.Model).IsZero() {
errMsg := fmt.Errorf("model definition not found in imported package. Model definitions often use the filename `model.json`, but are not required to have this filename. One and exactly one entity containing schema: model.core must be present, otherwise the model package is considered malformed")
regErrStore.InsertEntityRegError("", "", entity.Model, filename, errMsg)
return pkg, errMsg
}

return pkg, nil
}

func processDir(dirPath string, pkg *PackagingUnit, regErrStore RegistrationErrorStore) error {
var tempDirs []string
defer func() {
for _, tempDir := range tempDirs {
os.RemoveAll(tempDir)
}
}()

return filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
err = utils.ErrFileWalkDir(fmt.Errorf("error accessing path: %w", err), path)
regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
regErrStore.AddInvalidDefinition(path, err)
return nil
}
if f.IsDir() {
if info.IsDir() {
return nil
}
byt, _ := os.ReadFile(path)
if byt == nil {

// Read the file content
data, err := os.ReadFile(path)
if err != nil {
err = oci.ErrReadingFile(err)
regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
regErrStore.AddInvalidDefinition(path, err)
return nil
}

var e entity.Entity
e, err = getEntity(byt)
// Check if the file is an OCI artifact
if oci.IsOCIArtifact(data) {
// Extract the OCI artifact
tempDir, err := oci.CreateTempOCIContentDir()
if err != nil {
regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
regErrStore.AddInvalidDefinition(path, err)
return nil
}
tempDirs = append(tempDirs, tempDir)
err = oci.UnCompressOCIArtifact(path, tempDir)
if err != nil {
regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
regErrStore.AddInvalidDefinition(path, err)
return nil
}
// Recursively process the extracted directory
if err := processDir(tempDir, pkg, regErrStore); err != nil {
return err
}
return nil
}

// Check if the file is a zip or tar file
if utils.IsZip(path) || utils.IsTarGz(path) {
tempDir, err := os.MkdirTemp("", "nested-extract-")
if err != nil {
err = utils.ErrCreateDir(fmt.Errorf("error creating temp directory for nested archive extraction: %w", err), tempDir)
regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
regErrStore.AddInvalidDefinition(path, err)
return nil
}
tempDirs = append(tempDirs, tempDir)
if err := utils.ExtractFile(path, tempDir); err != nil {
regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
regErrStore.AddInvalidDefinition(path, err)
return nil
}
// Recursively process the extracted directory
if err := processDir(tempDir, pkg, regErrStore); err != nil {
return err
}
return nil
}

content := data
content, err = utils.YAMLToJSON(content)
if err != nil {
regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
return nil
}
// Determine the entity type
entityType, err := utils.FindEntityType(content)
if err != nil {
regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
regErrStore.AddInvalidDefinition(path, err)
return nil
}

// set it to pkgunit
if entityType == "" {
// Not an entity we care about
return nil
}

// Get the entity
var e entity.Entity
e, err = getEntity(content)
if err != nil {
regErrStore.InsertEntityRegError("", "", entityType, filepath.Base(path), fmt.Errorf("could not get entity: %w", err))
regErrStore.AddInvalidDefinition(path, fmt.Errorf("could not get entity: %w", err))
return nil
}

// Add the entity to the packaging unit
switch e.Type() {
case entity.Model:
if !reflect.ValueOf(pkg.model).IsZero() {
// currently models inside models are not handled
return nil
}
model, err := utils.Cast[*model.ModelDefinition](e)
if err != nil {
modelName := ""
if model != nil {
modelName = model.Name
}
regErrStore.InsertEntityRegError("", modelName, entityType, modelName, ErrGetEntity(err))
regErrStore.AddInvalidDefinition(path, ErrGetEntity(err))
return nil
}
pkg.model = *model
pkg.Model = *model
case entity.ComponentDefinition:
comp, err := utils.Cast[*component.ComponentDefinition](e)
if err != nil {
componentName := ""
if comp != nil {
componentName = comp.Component.Kind
}
regErrStore.InsertEntityRegError("", "", entityType, componentName, ErrGetEntity(err))
regErrStore.AddInvalidDefinition(path, ErrGetEntity(err))
return nil
}
pkg.components = append(pkg.components, *comp)
pkg.Components = append(pkg.Components, *comp)
case entity.RelationshipDefinition:
rel, err := utils.Cast[*relationship.RelationshipDefinition](e)
if err != nil {
relationshipName := ""
if rel != nil {
relationshipName = rel.Model.Name
}
regErrStore.InsertEntityRegError("", "", entityType, relationshipName, ErrGetEntity(err))
regErrStore.AddInvalidDefinition(path, ErrGetEntity(err))
return nil
}
pkg.relationships = append(pkg.relationships, *rel)
pkg.Relationships = append(pkg.Relationships, *rel)
default:
// Unhandled entity type
return nil
}
return nil
})
if err != nil {
return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("Could not completely walk the file tree: %e", err))
}
if reflect.ValueOf(pkg.model).IsZero() {
err := fmt.Errorf("Model definition not found in imported package. Model definitions often use the filename `model.json`, but are not required to have this filename. One and exactly one entity containing schema: model.core....... ...... must be present, otherwise the model package is considered malformed..")
regErrStore.AddInvalidDefinition(d.dirpath, err)
return pkg, err
}
return pkg, err
}
4 changes: 2 additions & 2 deletions models/registration/interface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@ type RegistrationErrorStore interface {
InsertEntityRegError(hostname string, modelName string, entityType entity.EntityType, entityName string, err error)
}

// Anything that can be parsed into a packagingUnit is a RegisterableEntity in Meshery server
// Anything that can be parsed into a PackagingUnit is a RegisterableEntity in Meshery server
type RegisterableEntity interface {
/*
1. `err` - this is a breaking error, which signifies that the given entity is invalid and cannot be registered
2. Errors encountered while parsing items into meshmodel entites are stored in the RegistrationErrorStore
*/
PkgUnit(RegistrationErrorStore) (packagingUnit, error)
PkgUnit(RegistrationErrorStore) (PackagingUnit, error)
}
4 changes: 2 additions & 2 deletions models/registration/oci.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ type OCIImage struct {
_ gcrv1.Image
}

func (o OCIImage) PkgUnit(regErrStore RegistrationErrorStore) (packagingUnit, error) {
pkg := packagingUnit{}
func (o OCIImage) PkgUnit(regErrStore RegistrationErrorStore) (PackagingUnit, error) {
pkg := PackagingUnit{}
return pkg, nil
}
Loading

0 comments on commit 1366959

Please sign in to comment.