Skip to content

Commit

Permalink
feat(export): With no filename, write zip bytes to stdout or API
Browse files Browse the repository at this point in the history
  • Loading branch information
dustmop committed May 13, 2020
1 parent 25fc08c commit 880ee63
Show file tree
Hide file tree
Showing 8 changed files with 189 additions and 81 deletions.
25 changes: 21 additions & 4 deletions api/datasets.go
Original file line number Diff line number Diff line change
Expand Up @@ -281,11 +281,19 @@ func (h *DatasetHandlers) listHandler(w http.ResponseWriter, r *http.Request) {
// if we are in read-only mode, we should error,
// otherwise, resolve the peername and proceed as normal
func (h *DatasetHandlers) getHandler(w http.ResponseWriter, r *http.Request) {
ref, err := dsref.Parse(HTTPPathToQriPath(r.URL.Path))
if err != nil {
util.WriteErrResponse(w, http.StatusBadRequest, err)
return
}

format := r.FormValue("format")
p := lib.GetParams{
Refstr: HTTPPathToQriPath(r.URL.Path),
Refstr: ref.String(),
Format: format,
}
res := lib.GetResult{}
err := h.Get(&p, &res)
err = h.Get(&p, &res)
if err != nil {
if err == repo.ErrNoHistory {
util.WriteErrResponse(w, http.StatusUnprocessableEntity, err)
Expand All @@ -300,8 +308,17 @@ func (h *DatasetHandlers) getHandler(w http.ResponseWriter, r *http.Request) {
return
}

// Handle getting a zip file as binary data
if format == "zip" {
zipFilename := fmt.Sprintf("%s.zip", ref.Name)
w.Header().Set("Content-Type", extensionToMimeType(".zip"))
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", zipFilename))
w.Write(res.Bytes)
return
}

// TODO (b5) - remove this. res.Ref should be used instead
ref := reporef.DatasetRef{
datasetRef := reporef.DatasetRef{
Peername: res.Dataset.Peername,
ProfileID: profile.IDB58DecodeOrEmpty(res.Dataset.ProfileID),
Name: res.Dataset.Name,
Expand All @@ -310,7 +327,7 @@ func (h *DatasetHandlers) getHandler(w http.ResponseWriter, r *http.Request) {
Published: res.Published,
Dataset: res.Dataset,
}
util.WriteResponse(w, ref)
util.WriteResponse(w, datasetRef)
}

func (h *DatasetHandlers) diffHandler(w http.ResponseWriter, r *http.Request) {
Expand Down
43 changes: 2 additions & 41 deletions api/fsi_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
"github.com/qri-io/dataset"
"github.com/qri-io/qri/fsi"
"github.com/qri-io/qri/lib"
"github.com/qri-io/qri/p2p"
reporef "github.com/qri-io/qri/repo/ref"
)

Expand Down Expand Up @@ -80,46 +79,6 @@ func TestFSIHandlers(t *testing.T) {
runHandlerTestCases(t, "checkout", h.CheckoutHandler(""), checkoutCases, true)
}

type APITestRunner struct {
Node *p2p.QriNode
NodeTeardown func()
Inst *lib.Instance
TmpDir string
WorkDir string
PrevXformVer string
}

func NewAPITestRunner(t *testing.T) *APITestRunner {
run := APITestRunner{}
run.Node, run.NodeTeardown = newTestNode(t)
run.Inst = newTestInstanceWithProfileFromNode(run.Node)

tmpDir, err := ioutil.TempDir("", "api_test")
if err != nil {
t.Fatal(err)
}
run.TmpDir = tmpDir

run.PrevXformVer = APIVersion
APIVersion = "test_version"

return &run
}

func (r *APITestRunner) Delete() {
os.RemoveAll(r.TmpDir)
APIVersion = r.PrevXformVer
r.NodeTeardown()
}

func (r *APITestRunner) MustMakeWorkDir(t *testing.T, name string) string {
r.WorkDir = filepath.Join(r.TmpDir, name)
if err := os.MkdirAll(r.WorkDir, os.ModePerm); err != nil {
t.Fatal(err)
}
return r.WorkDir
}

// TODO (ramfox): this test should be split for each endpoint:
// getHandler
// rootHandler
Expand Down Expand Up @@ -275,6 +234,8 @@ func TestFSIWrite(t *testing.T) {
dsm := lib.NewDatasetMethods(inst)
fsiHandler := NewFSIHandlers(inst, false)

// TODO(dustmop): Use a TestRunner here, and have it call SaveDataset instead.

// Save version 1
saveParams := lib.SaveParams{
Ref: "me/write_test",
Expand Down
37 changes: 9 additions & 28 deletions api/render_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ import (
"testing"

"github.com/qri-io/dataset"
"github.com/qri-io/qri/lib"
reporef "github.com/qri-io/qri/repo/ref"
)

func TestRenderHandler(t *testing.T) {
Expand All @@ -24,35 +22,18 @@ func TestRenderHandler(t *testing.T) {
}

func TestRenderReadmeHandler(t *testing.T) {
node, teardown := newTestNode(t)
defer teardown()
run := NewAPITestRunner(t)
defer run.Delete()

inst := newTestInstanceWithProfileFromNode(node)
h := NewRenderHandlers(inst)
dsm := lib.NewDatasetMethods(inst)

// TODO(dlong): Copied from fsi_test, refactor into a common utility
saveParams := lib.SaveParams{
Ref: "me/render_readme_test",
Dataset: &dataset.Dataset{
Meta: &dataset.Meta{
Title: "title one",
},
Readme: &dataset.Readme{
ScriptBytes: []byte("# hi\n\ntest"),
},
},
BodyPath: "testdata/cities/data.csv",
}
res := reporef.DatasetRef{}
if err := dsm.Save(&saveParams, &res); err != nil {
t.Fatal(err)
}
// Save a version of the dataset
ds := run.BuildDataset("render_readme_test")
ds.Meta = &dataset.Meta{Title: "title one"}
ds.Readme = &dataset.Readme{ScriptBytes: []byte("# hi\n\ntest")}
run.SaveDataset(ds, "testdata/cities/data.csv")

// Render the dataset
actualStatusCode, actualBody := APICall(
"/render/peer/render_readme_test",
h.RenderHandler)
h := run.NewRenderHandlers()
actualStatusCode, actualBody := APICall("/render/peer/render_readme_test", h.RenderHandler)
if actualStatusCode != 200 {
t.Errorf("expected status code 200, got %d", actualStatusCode)
}
Expand Down
89 changes: 89 additions & 0 deletions api/test_runner_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package api

import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"
"time"

"github.com/qri-io/dataset"
"github.com/qri-io/qri/base/dsfs"
"github.com/qri-io/qri/lib"
"github.com/qri-io/qri/p2p"
reporef "github.com/qri-io/qri/repo/ref"
)

type APITestRunner struct {
Node *p2p.QriNode
NodeTeardown func()
Inst *lib.Instance
DsfsTsFunc func() time.Time
TmpDir string
WorkDir string
PrevXformVer string
}

func NewAPITestRunner(t *testing.T) *APITestRunner {
run := APITestRunner{}
run.Node, run.NodeTeardown = newTestNode(t)
run.Inst = newTestInstanceWithProfileFromNode(run.Node)

tmpDir, err := ioutil.TempDir("", "api_test")
if err != nil {
t.Fatal(err)
}
run.TmpDir = tmpDir

counter := 0
run.DsfsTsFunc = dsfs.Timestamp
dsfs.Timestamp = func() time.Time {
counter++
return time.Date(2001, 01, 01, 01, counter, 01, 01, time.UTC)
}

run.PrevXformVer = APIVersion
APIVersion = "test_version"

return &run
}

func (r *APITestRunner) Delete() {
os.RemoveAll(r.TmpDir)
APIVersion = r.PrevXformVer
r.NodeTeardown()
}

func (r *APITestRunner) MustMakeWorkDir(t *testing.T, name string) string {
r.WorkDir = filepath.Join(r.TmpDir, name)
if err := os.MkdirAll(r.WorkDir, os.ModePerm); err != nil {
t.Fatal(err)
}
return r.WorkDir
}

func (r *APITestRunner) BuildDataset(dsName string) *dataset.Dataset {
ds := dataset.Dataset{
Peername: "peer",
Name: dsName,
}
return &ds
}

func (r *APITestRunner) SaveDataset(ds *dataset.Dataset, bodyFilename string) {
dsm := lib.NewDatasetMethods(r.Inst)
saveParams := lib.SaveParams{
Ref: fmt.Sprintf("peer/%s", ds.Name),
Dataset: ds,
BodyPath: bodyFilename,
}
res := reporef.DatasetRef{}
if err := dsm.Save(&saveParams, &res); err != nil {
panic(err)
}
}

func (r *APITestRunner) NewRenderHandlers() *RenderHandlers {
return NewRenderHandlers(r.Inst)
}
Binary file modified api/testdata/api.snapshot
Binary file not shown.
Binary file added api/testdata/cities/exported.zip
Binary file not shown.
36 changes: 36 additions & 0 deletions api/zip_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package api

import (
"io/ioutil"
"testing"

"github.com/google/go-cmp/cmp"
"github.com/qri-io/dataset"
)

func TestGetZip(t *testing.T) {
run := NewAPITestRunner(t)
defer run.Delete()

// Save a version of the dataset
ds := run.BuildDataset("test_ds")
ds.Meta = &dataset.Meta{Title: "some title"}
ds.Readme = &dataset.Readme{ScriptBytes: []byte("# hi\n\nthis is a readme")}
run.SaveDataset(ds, "testdata/cities/data.csv")

// Get a zip file binary over the API
dsHandler := NewDatasetHandlers(run.Inst, false)
gotStatusCode, gotBodyString := APICall("/peer/test_ds?format=zip", dsHandler.GetHandler)
if gotStatusCode != 200 {
t.Fatalf("expected status code 200, got %d", gotStatusCode)
}

// Compare the API response to the expected zip file
expectBytes, err := ioutil.ReadFile("testdata/cities/exported.zip")
if err != nil {
t.Fatalf("error reading expected bytes: %s", err)
}
if diff := cmp.Diff(string(expectBytes), gotBodyString); diff != "" {
t.Errorf("byte mismatch (-want +got):\n%s", diff)
}
}
40 changes: 32 additions & 8 deletions lib/datasets.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package lib

import (
"bytes"
"context"
"encoding/json"
"fmt"
Expand Down Expand Up @@ -186,7 +187,7 @@ func (m *DatasetMethods) ListRawRefs(p *ListParams, text *string) error {
return err
}

// GetParams defines parameters for looking up the body of a dataset
// GetParams defines parameters for looking up the head or body of a dataset
type GetParams struct {
// Refstr to get, representing a dataset ref to be parsed
Refstr string
Expand All @@ -200,7 +201,9 @@ type GetParams struct {
Limit, Offset int
All bool

Outfile string
// outfile is a filename to save the dataset to
Outfile string
// whether to generate a filename from the dataset name instead
GenFilename bool
}

Expand All @@ -222,6 +225,10 @@ type GetResult struct {
// then res.Bytes is loaded with the body. If the selector is "stats", then res.Bytes is loaded
// with the generated stats.
func (m *DatasetMethods) Get(p *GetParams, res *GetResult) error {
if err := qfs.AbsPath(&p.Outfile); err != nil {
return err
}

if m.inst.rpc != nil {
return checkRPCError(m.inst.rpc.Call("DatasetMethods.Get", p, res))
}
Expand Down Expand Up @@ -284,13 +291,21 @@ func (m *DatasetMethods) Get(p *GetParams, res *GetResult) error {
}

if p.Format == "zip" {
// Only if GenFilename is true, and no output filename is set, generate one from the
// dataset name
if p.Outfile == "" && p.GenFilename {
p.Outfile = fmt.Sprintf("%s.zip", ds.Name)
}
// TODO(dustmop): Abs to handle rpc
zipFile, err := os.Create(p.Outfile)
if err != nil {
return err
var outBuf bytes.Buffer
var zipFile io.Writer
if p.Outfile == "" {
// In this case, write to a buffer, which will be assigned to res.Bytes later on
zipFile = &outBuf
} else {
zipFile, err = os.Create(p.Outfile)
if err != nil {
return err
}
}
currRef := dsref.Ref{Username: ds.Peername, Name: ds.Name}
// TODO(dustmop): This function is inefficient and a poor use of logbook, but it's
Expand All @@ -300,8 +315,17 @@ func (m *DatasetMethods) Get(p *GetParams, res *GetResult) error {
return err
}
err = archive.WriteZip(ctx, m.inst.repo.Store(), ds, "json", initID, currRef, zipFile)
res.Message = fmt.Sprintf("Wrote archive %s", p.Outfile)
return err
if err != nil {
return err
}
// Handle output. If outfile is empty, return the raw bytes. Otherwise provide a helpful
// message for the user
if p.Outfile == "" {
res.Bytes = outBuf.Bytes()
} else {
res.Message = fmt.Sprintf("Wrote archive %s", p.Outfile)
}
return nil
}

if p.Selector == "body" {
Expand Down

0 comments on commit 880ee63

Please sign in to comment.