Skip to content

Commit

Permalink
feat(ConvertBodyFile): extracted actions.ConvertBodyFile from LookupB…
Browse files Browse the repository at this point in the history 
…ody func
  • Loading branch information
@b5
b5 committed Sep 30, 2018
1 parent 8c39e7b commit d8a7f77
Show file tree
Hide file tree
Showing 2 changed files with 166 additions and 99 deletions.
20 changes: 16 additions & 4 deletions actions/body.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,24 @@ func LookupBody(node *p2p.QriNode, path string, format dataset.DataFormat, fcfg
Schema: ds.Structure.Schema,
})

buf, err := dsio.NewEntryBuffer(st)
data, err = ConvertBodyFile(file, ds.Structure, st, limit, offset, all)
if err != nil {
log.Debug(err.Error())
return "", nil, err
}

return ds.BodyPath, data, nil
}

// ConvertBodyFile takes an input file & structure, and converts a specified selection
// to the structure specified by out
func ConvertBodyFile(file cafs.File, in, out *dataset.Structure, limit, offset int, all bool) (data []byte, err error) {
buf, err := dsio.NewEntryBuffer(out)
if err != nil {
err = fmt.Errorf("error allocating result buffer: %s", err)
return
}
rr, err := dsio.NewEntryReader(ds.Structure, file)
rr, err := dsio.NewEntryReader(in, file)
if err != nil {
err = fmt.Errorf("error allocating data reader: %s", err)
return
Expand All @@ -58,8 +70,8 @@ func LookupBody(node *p2p.QriNode, path string, format dataset.DataFormat, fcfg
err = dsio.Copy(rr, buf)

if err := buf.Close(); err != nil {
return "", nil, fmt.Errorf("error closing row buffer: %s", err.Error())
return nil, fmt.Errorf("error closing row buffer: %s", err.Error())
}

return ds.BodyPath, buf.Bytes(), nil
return buf.Bytes(), nil
}
245 changes: 150 additions & 95 deletions api/datasets.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,13 @@ import (
"path/filepath"
"strings"

"github.com/qri-io/ioes"

util "github.com/datatogether/api/apiutil"
"github.com/qri-io/cafs"
"github.com/qri-io/dataset"
"github.com/qri-io/dataset/dsutil"
"github.com/qri-io/dsdiff"
"github.com/qri-io/ioes"
"github.com/qri-io/qri/actions"
"github.com/qri-io/qri/lib"
"github.com/qri-io/qri/p2p"
"github.com/qri-io/qri/repo"
Expand Down Expand Up @@ -356,95 +357,12 @@ func (h *DatasetHandlers) initHandler(w http.ResponseWriter, r *http.Request) {
}

default:
datafile, dataHeader, err := r.FormFile("file")
if err != nil && err != http.ErrMissingFile {
util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("error opening dataset file: %s", err))
return
}
if datafile != nil {
switch strings.ToLower(filepath.Ext(dataHeader.Filename)) {
case ".yaml", ".yml":
data, err := ioutil.ReadAll(datafile)
if err != nil {
util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("error reading dataset file: %s", err))
return
}
if err = dsutil.UnmarshalYAMLDatasetPod(data, dsp); err != nil {
util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("error unmarshaling yaml file: %s", err))
return
}
case ".json":
if err = json.NewDecoder(datafile).Decode(dsp); err != nil {
util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("error decoding json file: %s", err))
return
}
}
}

tfFile, _, err := r.FormFile("transform")
if err != nil && err != http.ErrMissingFile {
util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("error opening transform file: %s", err))
return
}
if tfFile != nil {
// TODO - this assumes a skylark / starlark transform file
f, err := ioutil.TempFile("", "transform")
if err != nil {
util.WriteErrResponse(w, http.StatusBadRequest, err)
return
}
defer os.Remove(f.Name())
io.Copy(f, tfFile)
if dsp.Transform == nil {
dsp.Transform = &dataset.TransformPod{}
}
dsp.Transform.Syntax = "skylark"
dsp.Transform.ScriptPath = f.Name()
}

vizFile, _, err := r.FormFile("viz")
if err != nil && err != http.ErrMissingFile {
util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("error opening viz file: %s", err))
return
}
if vizFile != nil {
// TODO - this assumes an html viz file
f, err := ioutil.TempFile("", "viz")
if err != nil {
util.WriteErrResponse(w, http.StatusBadRequest, err)
return
}
defer os.Remove(f.Name())
io.Copy(f, vizFile)
if dsp.Viz == nil {
dsp.Viz = &dataset.Viz{}
}
dsp.Viz.Format = "html"
dsp.Viz.ScriptPath = f.Name()
}

dsp.Peername = r.FormValue("peername")
dsp.Name = r.FormValue("name")
dsp.BodyPath = r.FormValue("body_path")

bodyfile, bodyHeader, err := r.FormFile("body")
if err != nil && err != http.ErrMissingFile {
util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("error opening body file: %s", err))
cleanup, err := formFileDataset(dsp, r)
if err != nil {
util.WriteErrResponse(w, http.StatusBadRequest, err)
return
}
if bodyfile != nil {
path := filepath.Join(os.TempDir(), bodyHeader.Filename)
f, err := os.Create(path)
if err != nil {
util.WriteErrResponse(w, http.StatusInternalServerError, fmt.Errorf("error writing body file: %s", err.Error()))
return
}
defer os.Remove(path)
io.Copy(f, bodyfile)
f.Close()
dsp.BodyPath = path
}

defer cleanup()
}

// TODO - fix this awful mess, ioes needs some method for piping it's output
Expand Down Expand Up @@ -473,15 +391,10 @@ func (h *DatasetHandlers) initHandler(w http.ResponseWriter, r *http.Request) {
}

if p.ReturnBody {
// TODO - this'll only work for JSON responses
data, err := ioutil.ReadAll(res.Dataset.Body.(io.Reader))
if err != nil {
log.Info(err.Error())
if err := addBodyFile(res); err != nil {
util.WriteErrResponse(w, http.StatusInternalServerError, err)
return
}

res.Dataset.Body = json.RawMessage(data)
}

// util.WriteResponse(w, res)
Expand All @@ -497,6 +410,141 @@ func (h *DatasetHandlers) initHandler(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(env)
}

func formFileDataset(dsp *dataset.DatasetPod, r *http.Request) (cleanup func(), err error) {
var rmFiles []*os.File
cleanup = func() {
for _, f := range rmFiles {
// TODO - log error?
os.Remove(f.Name())
}
}

datafile, dataHeader, err := r.FormFile("file")
if err != nil && err != http.ErrMissingFile {
err = fmt.Errorf("error opening dataset file: %s", err)
return
}
if datafile != nil {
switch strings.ToLower(filepath.Ext(dataHeader.Filename)) {
case ".yaml", ".yml":
var data []byte
data, err = ioutil.ReadAll(datafile)
if err != nil {
err = fmt.Errorf("error reading dataset file: %s", err)
return
}
if err = dsutil.UnmarshalYAMLDatasetPod(data, dsp); err != nil {
err = fmt.Errorf("error unmarshaling yaml file: %s", err)
return
}
case ".json":
if err = json.NewDecoder(datafile).Decode(dsp); err != nil {
err = fmt.Errorf("error decoding json file: %s", err)
return
}
}
}

tfFile, _, err := r.FormFile("transform")
if err != nil && err != http.ErrMissingFile {
err = fmt.Errorf("error opening transform file: %s", err)
return
}
if tfFile != nil {
// TODO - this assumes a skylark / starlark transform file
f, e := ioutil.TempFile("", "transform")
if e != nil {
err = e
return
}
rmFiles = append(rmFiles, f)
io.Copy(f, tfFile)
if dsp.Transform == nil {
dsp.Transform = &dataset.TransformPod{}
}
dsp.Transform.Syntax = "skylark"
dsp.Transform.ScriptPath = f.Name()
}

vizFile, _, err := r.FormFile("viz")
if err != nil && err != http.ErrMissingFile {
err = fmt.Errorf("error opening viz file: %s", err)
return
}
if vizFile != nil {
// TODO - this assumes an html viz file
f, e := ioutil.TempFile("", "viz")
if e != nil {
err = e
return
}
rmFiles = append(rmFiles, f)
io.Copy(f, vizFile)
if dsp.Viz == nil {
dsp.Viz = &dataset.Viz{}
}
dsp.Viz.Format = "html"
dsp.Viz.ScriptPath = f.Name()
}

dsp.Peername = r.FormValue("peername")
dsp.Name = r.FormValue("name")
dsp.BodyPath = r.FormValue("body_path")

bodyfile, bodyHeader, err := r.FormFile("body")
if err != nil && err != http.ErrMissingFile {
err = fmt.Errorf("error opening body file: %s", err)
return
}
if bodyfile != nil {
path := filepath.Join(os.TempDir(), bodyHeader.Filename)
f, e := os.Create(path)
if e != nil {
err = fmt.Errorf("error writing body file: %s", e.Error())
return
}
rmFiles = append(rmFiles, f)
io.Copy(f, bodyfile)
f.Close()
dsp.BodyPath = path
}

return
}

func addBodyFile(res *repo.DatasetRef) error {
if res.Dataset.Structure.Format == dataset.JSONDataFormat.String() {
// TODO - this'll only work for JSON responses
data, err := ioutil.ReadAll(res.Dataset.Body.(io.Reader))
if err != nil {
return err
}
res.Dataset.Body = json.RawMessage(data)
} else {
if file, ok := res.Dataset.Body.(cafs.File); ok {
in := &dataset.Structure{}
if err := in.Decode(res.Dataset.Structure); err != nil {
return err
}

st := &dataset.Structure{}
st.Assign(in, &dataset.Structure{
Format: dataset.JSONDataFormat,
Schema: in.Schema,
})

data, err := actions.ConvertBodyFile(file, in, st, 0, 0, true)
if err != nil {
return fmt.Errorf("converting body file to JSON: %s", err)
}
res.Dataset.Body = json.RawMessage(data)
} else {
log.Error("response body isn't a cafs.File")
}
}
return nil
}

func (h *DatasetHandlers) addHandler(w http.ResponseWriter, r *http.Request) {
ref, err := DatasetRefFromPath(r.URL.Path[len("/add"):])
if err != nil {
Expand Down Expand Up @@ -540,6 +588,13 @@ func (h *DatasetHandlers) saveHandler(w http.ResponseWriter, r *http.Request) {
dsp.Name = args.Name
}
}
} else {
cleanup, err := formFileDataset(dsp, r)
if err != nil {
util.WriteErrResponse(w, http.StatusBadRequest, err)
return
}
defer cleanup()
}

res := &repo.DatasetRef{}
Expand Down

0 comments on commit d8a7f77

Please sign in to comment.