-
Notifications
You must be signed in to change notification settings - Fork 66
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix(validate): Improve validate so it works with FSI
FSI broke validate because we switched from schema.json back to structure.json. The old functionality was looking specifically for schema.json (in the cmd package, no less) and did not survive this change. Instead, we should use the FSI package which correctly finds relevant component files. Cleanup the lower levels of how validate works. Before, it was converting things back and forth, between cmd -> lib -> base -> dataset -> jsonschema. No need to do all of that unnecessary work.
- Loading branch information
Showing
9 changed files
with
142 additions
and
195 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,116 +1,46 @@ | ||
| package base | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "context" | ||
| "encoding/json" | ||
| "fmt" | ||
| "io/ioutil" | ||
|
|
||
| "github.com/qri-io/dataset" | ||
| "github.com/qri-io/dataset/detect" | ||
| "github.com/qri-io/dataset/dsio" | ||
| "github.com/qri-io/dataset/validate" | ||
| "github.com/qri-io/jsonschema" | ||
| "github.com/qri-io/qfs" | ||
| "github.com/qri-io/qri/base/dsfs" | ||
| "github.com/qri-io/qri/repo" | ||
| ) | ||
|
|
||
| // Validate checks a dataset body for errors based on a schema | ||
| func Validate(ctx context.Context, r repo.Repo, ref repo.DatasetRef, body, schema qfs.File) (errors []jsonschema.ValError, err error) { | ||
| if !ref.IsEmpty() { | ||
| err = repo.CanonicalizeDatasetRef(r, &ref) | ||
| if err != nil && err != repo.ErrNotFound { | ||
| log.Debug(err.Error()) | ||
| err = fmt.Errorf("error with new reference: %s", err.Error()) | ||
| return | ||
| } | ||
| // Validate checks a dataset body for errors based on the structure's schema | ||
| func Validate(ctx context.Context, r repo.Repo, body qfs.File, st *dataset.Structure) ([]jsonschema.ValError, error) { | ||
| if body == nil { | ||
| return nil, fmt.Errorf("body passed to Validate must not be nil") | ||
| } | ||
|
|
||
| var ( | ||
| st = &dataset.Structure{} | ||
| data []byte | ||
| ) | ||
|
|
||
| // if a dataset is specified, load it | ||
| if ref.Path != "" { | ||
| if err = ReadDataset(ctx, r, &ref); err != nil { | ||
| log.Debug(err.Error()) | ||
| return | ||
| } | ||
|
|
||
| ds := ref.Dataset | ||
| st = ds.Structure | ||
| } else if body == nil { | ||
| err = fmt.Errorf("cannot find dataset: %s", ref) | ||
| return | ||
| if st == nil { | ||
| return nil, fmt.Errorf("st passed to Validate must not be nil") | ||
| } | ||
|
|
||
| if body != nil { | ||
| data, err = ioutil.ReadAll(body) | ||
| if err != nil { | ||
| log.Debug(err.Error()) | ||
| err = fmt.Errorf("error reading data: %s", err.Error()) | ||
| return | ||
| } | ||
|
|
||
| // if no schema, detect one | ||
| if st.Schema == nil { | ||
| var df dataset.DataFormat | ||
| df, err = detect.ExtensionDataFormat(body.FileName()) | ||
| if err != nil { | ||
| err = fmt.Errorf("detecting data format: %s", err.Error()) | ||
| return | ||
| } | ||
| str, _, e := detect.FromReader(df, bytes.NewBuffer(data)) | ||
| if e != nil { | ||
| err = fmt.Errorf("error detecting from reader: %s", e) | ||
| return | ||
| } | ||
| st = str | ||
| // jsonschema assumes body is json, convert the format if necessary | ||
| if st.Format != "json" { | ||
| convert := dataset.Structure{ | ||
| Format: "json", | ||
| Schema: st.Schema, | ||
| } | ||
| } | ||
|
|
||
| // if a schema is specified, override with it | ||
| if schema != nil { | ||
| stbytes, e := ioutil.ReadAll(schema) | ||
| if e != nil { | ||
| log.Debug(e.Error()) | ||
| err = e | ||
| return | ||
| } | ||
| sch := map[string]interface{}{} | ||
| if e := json.Unmarshal(stbytes, &sch); e != nil { | ||
| err = fmt.Errorf("error reading schema: %s", e.Error()) | ||
| return | ||
| } | ||
| st.Schema = sch | ||
| } | ||
|
|
||
| if data == nil && ref.Dataset != nil { | ||
| ds := ref.Dataset | ||
|
|
||
| f, e := dsfs.LoadBody(ctx, r.Store(), ds) | ||
| if e != nil { | ||
| log.Debug(e.Error()) | ||
| err = fmt.Errorf("error loading dataset data: %s", e.Error()) | ||
| return | ||
| } | ||
| data, err = ioutil.ReadAll(f) | ||
| file, err := ConvertBodyFormat(body, st, &convert) | ||
| if err != nil { | ||
| log.Debug(err.Error()) | ||
| err = fmt.Errorf("error loading dataset data: %s", err.Error()) | ||
| return | ||
| return nil, err | ||
| } | ||
| body = file | ||
| } | ||
|
|
||
| er, err := dsio.NewEntryReader(st, bytes.NewBuffer(data)) | ||
| // jsonschema does not handle data streams, have to read the whole body | ||
| data, err := ioutil.ReadAll(body) | ||
| if err != nil { | ||
| log.Debug(err.Error()) | ||
| err = fmt.Errorf("error reading data: %s", err.Error()) | ||
| return | ||
| return nil, err | ||
| } | ||
|
|
||
| return validate.EntryReader(er) | ||
| jsch, err := st.JSONSchema() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return jsch.ValidateBytes(data) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.