diff --git a/actions/body.go b/actions/body.go index fcef61366..065342ad3 100644 --- a/actions/body.go +++ b/actions/body.go @@ -37,12 +37,24 @@ func LookupBody(node *p2p.QriNode, path string, format dataset.DataFormat, fcfg Schema: ds.Structure.Schema, }) - buf, err := dsio.NewEntryBuffer(st) + data, err = ConvertBodyFile(file, ds.Structure, st, limit, offset, all) + if err != nil { + log.Debug(err.Error()) + return "", nil, err + } + + return ds.BodyPath, data, nil +} + +// ConvertBodyFile takes an input file & structure, and converts a specified selection +// to the structure specified by out +func ConvertBodyFile(file cafs.File, in, out *dataset.Structure, limit, offset int, all bool) (data []byte, err error) { + buf, err := dsio.NewEntryBuffer(out) if err != nil { err = fmt.Errorf("error allocating result buffer: %s", err) return } - rr, err := dsio.NewEntryReader(ds.Structure, file) + rr, err := dsio.NewEntryReader(in, file) if err != nil { err = fmt.Errorf("error allocating data reader: %s", err) return @@ -58,8 +70,8 @@ func LookupBody(node *p2p.QriNode, path string, format dataset.DataFormat, fcfg err = dsio.Copy(rr, buf) if err := buf.Close(); err != nil { - return "", nil, fmt.Errorf("error closing row buffer: %s", err.Error()) + return nil, fmt.Errorf("error closing row buffer: %s", err.Error()) } - return ds.BodyPath, buf.Bytes(), nil + return buf.Bytes(), nil } diff --git a/api/datasets.go b/api/datasets.go index 060737d64..1841d1e67 100644 --- a/api/datasets.go +++ b/api/datasets.go @@ -12,12 +12,13 @@ import ( "path/filepath" "strings" - "github.com/qri-io/ioes" - util "github.com/datatogether/api/apiutil" + "github.com/qri-io/cafs" "github.com/qri-io/dataset" "github.com/qri-io/dataset/dsutil" "github.com/qri-io/dsdiff" + "github.com/qri-io/ioes" + "github.com/qri-io/qri/actions" "github.com/qri-io/qri/lib" "github.com/qri-io/qri/p2p" "github.com/qri-io/qri/repo" @@ -356,95 +357,12 @@ func (h *DatasetHandlers) initHandler(w http.ResponseWriter, r *http.Request) { } default: - datafile, dataHeader, err := r.FormFile("file") - if err != nil && err != http.ErrMissingFile { - util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("error opening dataset file: %s", err)) - return - } - if datafile != nil { - switch strings.ToLower(filepath.Ext(dataHeader.Filename)) { - case ".yaml", ".yml": - data, err := ioutil.ReadAll(datafile) - if err != nil { - util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("error reading dataset file: %s", err)) - return - } - if err = dsutil.UnmarshalYAMLDatasetPod(data, dsp); err != nil { - util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("error unmarshaling yaml file: %s", err)) - return - } - case ".json": - if err = json.NewDecoder(datafile).Decode(dsp); err != nil { - util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("error decoding json file: %s", err)) - return - } - } - } - - tfFile, _, err := r.FormFile("transform") - if err != nil && err != http.ErrMissingFile { - util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("error opening transform file: %s", err)) - return - } - if tfFile != nil { - // TODO - this assumes a skylark / starlark transform file - f, err := ioutil.TempFile("", "transform") - if err != nil { - util.WriteErrResponse(w, http.StatusBadRequest, err) - return - } - defer os.Remove(f.Name()) - io.Copy(f, tfFile) - if dsp.Transform == nil { - dsp.Transform = &dataset.TransformPod{} - } - dsp.Transform.Syntax = "skylark" - dsp.Transform.ScriptPath = f.Name() - } - - vizFile, _, err := r.FormFile("viz") - if err != nil && err != http.ErrMissingFile { - util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("error opening viz file: %s", err)) - return - } - if vizFile != nil { - // TODO - this assumes an html viz file - f, err := ioutil.TempFile("", "viz") - if err != nil { - util.WriteErrResponse(w, http.StatusBadRequest, err) - return - } - defer os.Remove(f.Name()) - io.Copy(f, vizFile) - if dsp.Viz == nil { - dsp.Viz = &dataset.Viz{} - } - dsp.Viz.Format = "html" - dsp.Viz.ScriptPath = f.Name() - } - - dsp.Peername = r.FormValue("peername") - dsp.Name = r.FormValue("name") - dsp.BodyPath = r.FormValue("body_path") - - bodyfile, bodyHeader, err := r.FormFile("body") - if err != nil && err != http.ErrMissingFile { - util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("error opening body file: %s", err)) + cleanup, err := formFileDataset(dsp, r) + if err != nil { + util.WriteErrResponse(w, http.StatusBadRequest, err) return } - if bodyfile != nil { - path := filepath.Join(os.TempDir(), bodyHeader.Filename) - f, err := os.Create(path) - if err != nil { - util.WriteErrResponse(w, http.StatusInternalServerError, fmt.Errorf("error writing body file: %s", err.Error())) - return - } - defer os.Remove(path) - io.Copy(f, bodyfile) - f.Close() - dsp.BodyPath = path - } - + defer cleanup() } // TODO - fix this awful mess, ioes needs some method for piping it's output @@ -473,15 +391,10 @@ func (h *DatasetHandlers) initHandler(w http.ResponseWriter, r *http.Request) { } if p.ReturnBody { - // TODO - this'll only work for JSON responses - data, err := ioutil.ReadAll(res.Dataset.Body.(io.Reader)) - if err != nil { - log.Info(err.Error()) + if err := addBodyFile(res); err != nil { util.WriteErrResponse(w, http.StatusInternalServerError, err) return } - - res.Dataset.Body = json.RawMessage(data) } // util.WriteResponse(w, res) @@ -497,6 +410,141 @@ func (h *DatasetHandlers) initHandler(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(env) } +func formFileDataset(dsp *dataset.DatasetPod, r *http.Request) (cleanup func(), err error) { + var rmFiles []*os.File + cleanup = func() { + for _, f := range rmFiles { + // TODO - log error? + os.Remove(f.Name()) + } + } + + datafile, dataHeader, err := r.FormFile("file") + if err != nil && err != http.ErrMissingFile { + err = fmt.Errorf("error opening dataset file: %s", err) + return + } + if datafile != nil { + switch strings.ToLower(filepath.Ext(dataHeader.Filename)) { + case ".yaml", ".yml": + var data []byte + data, err = ioutil.ReadAll(datafile) + if err != nil { + err = fmt.Errorf("error reading dataset file: %s", err) + return + } + if err = dsutil.UnmarshalYAMLDatasetPod(data, dsp); err != nil { + err = fmt.Errorf("error unmarshaling yaml file: %s", err) + return + } + case ".json": + if err = json.NewDecoder(datafile).Decode(dsp); err != nil { + err = fmt.Errorf("error decoding json file: %s", err) + return + } + } + } + + tfFile, _, err := r.FormFile("transform") + if err != nil && err != http.ErrMissingFile { + err = fmt.Errorf("error opening transform file: %s", err) + return + } + if tfFile != nil { + // TODO - this assumes a skylark / starlark transform file + f, e := ioutil.TempFile("", "transform") + if e != nil { + err = e + return + } + rmFiles = append(rmFiles, f) + io.Copy(f, tfFile) + if dsp.Transform == nil { + dsp.Transform = &dataset.TransformPod{} + } + dsp.Transform.Syntax = "skylark" + dsp.Transform.ScriptPath = f.Name() + } + + vizFile, _, err := r.FormFile("viz") + if err != nil && err != http.ErrMissingFile { + err = fmt.Errorf("error opening viz file: %s", err) + return + } + if vizFile != nil { + // TODO - this assumes an html viz file + f, e := ioutil.TempFile("", "viz") + if e != nil { + err = e + return + } + rmFiles = append(rmFiles, f) + io.Copy(f, vizFile) + if dsp.Viz == nil { + dsp.Viz = &dataset.Viz{} + } + dsp.Viz.Format = "html" + dsp.Viz.ScriptPath = f.Name() + } + + dsp.Peername = r.FormValue("peername") + dsp.Name = r.FormValue("name") + dsp.BodyPath = r.FormValue("body_path") + + bodyfile, bodyHeader, err := r.FormFile("body") + if err != nil && err != http.ErrMissingFile { + err = fmt.Errorf("error opening body file: %s", err) + return + } + if bodyfile != nil { + path := filepath.Join(os.TempDir(), bodyHeader.Filename) + f, e := os.Create(path) + if e != nil { + err = fmt.Errorf("error writing body file: %s", e.Error()) + return + } + rmFiles = append(rmFiles, f) + io.Copy(f, bodyfile) + f.Close() + dsp.BodyPath = path + } + + return +} + +func addBodyFile(res *repo.DatasetRef) error { + if res.Dataset.Structure.Format == dataset.JSONDataFormat.String() { + // TODO - this'll only work for JSON responses + data, err := ioutil.ReadAll(res.Dataset.Body.(io.Reader)) + if err != nil { + return err + } + res.Dataset.Body = json.RawMessage(data) + } else { + if file, ok := res.Dataset.Body.(cafs.File); ok { + in := &dataset.Structure{} + if err := in.Decode(res.Dataset.Structure); err != nil { + return err + } + + st := &dataset.Structure{} + st.Assign(in, &dataset.Structure{ + Format: dataset.JSONDataFormat, + Schema: in.Schema, + }) + + data, err := actions.ConvertBodyFile(file, in, st, 0, 0, true) + if err != nil { + return fmt.Errorf("converting body file to JSON: %s", err) + } + res.Dataset.Body = json.RawMessage(data) + } else { + log.Error("response body isn't a cafs.File") + } + } + return nil +} + func (h *DatasetHandlers) addHandler(w http.ResponseWriter, r *http.Request) { ref, err := DatasetRefFromPath(r.URL.Path[len("/add"):]) if err != nil { @@ -540,6 +588,13 @@ func (h *DatasetHandlers) saveHandler(w http.ResponseWriter, r *http.Request) { dsp.Name = args.Name } } + } else { + cleanup, err := formFileDataset(dsp, r) + if err != nil { + util.WriteErrResponse(w, http.StatusBadRequest, err) + return + } + defer cleanup() } res := &repo.DatasetRef{}