feat(dry_run flag): added dry run flag to creating & updating datasets

actions/actions_test.go

@@ -66,7 +66,7 @@ func addCitiesDataset(t *testing.T, node *p2p.QriNode) repo.DatasetRef {
  t.Fatal(err.Error())
  }
 
- ref, err := CreateDataset(node, tc.Name, tc.Input, tc.BodyFile(), nil, true)
+ ref, _, err := CreateDataset(node, tc.Name, tc.Input, tc.BodyFile(), nil, false, true)
  if err != nil {
  t.Fatal(err.Error())
  }
@@ -79,7 +79,7 @@ func addFlourinatedCompoundsDataset(t *testing.T, node *p2p.QriNode) repo.Datase
  t.Fatal(err.Error())
  }
 
- ref, err := CreateDataset(node, tc.Name, tc.Input, tc.BodyFile(), nil, true)
+ ref, _, err := CreateDataset(node, tc.Name, tc.Input, tc.BodyFile(), nil, false, true)
  if err != nil {
  t.Fatal(err.Error())
  }

actions/dataset.go

@@ -228,7 +228,7 @@ func UpdateDataset(node *p2p.QriNode, dsp *dataset.DatasetPod) (ds *dataset.Data
 }
 
 // CreateDataset initializes a dataset from a dataset pointer and data file
-func CreateDataset(node *p2p.QriNode, name string, ds *dataset.Dataset, data cafs.File, secrets map[string]string, pin bool) (ref repo.DatasetRef, err error) {
+func CreateDataset(node *p2p.QriNode, name string, ds *dataset.Dataset, data cafs.File, secrets map[string]string, dryRun, pin bool) (ref repo.DatasetRef, body cafs.File, err error) {
  var (
  r = node.Repo
  pro *profile.Profile
@@ -269,7 +269,16 @@ func CreateDataset(node *p2p.QriNode, name string, ds *dataset.Dataset, data caf
  return
  }
 
- if ref, err = repo.CreateDataset(node.Repo, name, ds, data, pin); err != nil {
+ if dryRun {
+ // dry-runs store to an in-memory repo
+ node.LocalStreams.Print("🏃🏽‍♀️ dry run\n")
+ r, err = repo.NewMemRepo(pro, cafs.NewMapstore(), profile.NewMemStore(), nil)
+ if err != nil {
+ return
+ }
+ }
+
+ if ref, err = repo.CreateDataset(r, name, ds, data, pin); err != nil {
  return
  }
 
@@ -281,6 +290,12 @@ func CreateDataset(node *p2p.QriNode, name string, ds *dataset.Dataset, data caf
  if pin && storeIsPinner {
  r.LogEvent(repo.ETDsPinned, ref)
  }
+
+ err = ReadDataset(r, &ref)
+ if err != nil {
+ body, err = r.Store().Get(datastore.NewKey(ref.Dataset.BodyPath))
+ }
+
  return
 }
 

actions/dataset_test.go

@@ -136,7 +136,7 @@ func createDataset(t *testing.T, rmf RepoMakerFunc) (*p2p.QriNode, repo.DatasetR
  return n, repo.DatasetRef{}
  }
 
- ref, err := CreateDataset(n, tc.Name, tc.Input, tc.BodyFile(), nil, true)
+ ref, _, err := CreateDataset(n, tc.Name, tc.Input, tc.BodyFile(), nil, false, true)
  if err != nil {
  t.Error(err.Error())
  }
@@ -200,7 +200,7 @@ func testDatasetPinning(t *testing.T, rmf RepoMakerFunc) {
  return
  }
 
- ref2, err := CreateDataset(node, tc.Name, tc.Input, tc.BodyFile(), nil, false)
+ ref2, _, err := CreateDataset(node, tc.Name, tc.Input, tc.BodyFile(), nil, false, false)
  if err != nil {
  t.Error(err.Error())
  return

actions/registry_test.go

@@ -31,7 +31,7 @@ func TestRegistry(t *testing.T) {
  return
  }
 
- ref, err := CreateDataset(node, tc.Name, tc.Input, tc.BodyFile(), nil, true)
+ ref, _, err := CreateDataset(node, tc.Name, tc.Input, tc.BodyFile(), nil, false, true)
  if err != nil {
  t.Fatal(err.Error())
 

actions/select_test.go

@@ -31,7 +31,7 @@ func TestSelect(t *testing.T) {
  return
  }
 
- ref, err := CreateDataset(node, tc.Name, tc.Input, tc.BodyFile(), nil, true)
+ ref, _, err := CreateDataset(node, tc.Name, tc.Input, tc.BodyFile(), nil, false, true)
  if err != nil {
  t.Fatal(err.Error())
 

actions/transform_test.go

@@ -39,6 +39,10 @@ def transform(qri):
  }
 
  ds := &dataset.Dataset{
+ Structure: &dataset.Structure{
+ Format: dataset.JSONDataFormat,
+ Schema: dataset.BaseSchemaArray,
+ },
  Transform: &dataset.Transform{
  Syntax: "skylark",
  ScriptPath: tfPath,

api/datasets.go

@@ -377,6 +377,28 @@ func (h *DatasetHandlers) initHandler(w http.ResponseWriter, r *http.Request) {
  }
  }
 
+ tfFile, _, err := r.FormFile("transform")
+ if err != nil && err != http.ErrMissingFile {
+ util.WriteErrResponse(w, http.StatusBadRequest, fmt.Errorf("error opening transform file: %s", err))
+ return
+ }
+ if tfFile != nil {
+ // TODO - this assumes a skylark / starlark transform file
+ f, err := ioutil.TempFile("", "transform")
+ if err != nil {
+ util.WriteErrResponse(w, http.StatusBadRequest, err)
+ return
+ }
+ defer os.Remove(f.Name())
+ io.Copy(f, tfFile)
+ if dsp.Transform == nil {
+ dsp.Transform = &dataset.TransformPod{
+ Syntax: "skylark",
+ ScriptPath: f.Name(),
+ }
+ }
+ }
+
  dsp.Peername = r.FormValue("peername")
  dsp.Name = r.FormValue("name")
  dsp.BodyPath = r.FormValue("body_path")
@@ -403,14 +425,25 @@ func (h *DatasetHandlers) initHandler(w http.ResponseWriter, r *http.Request) {
 
  res := &repo.DatasetRef{}
  p := &lib.SaveParams{
- Dataset: dsp,
- Private: r.FormValue("private") == "true",
+ Dataset: dsp,
+ Private: r.FormValue("private") == "true",
+ DryRun: r.FormValue("dry_run") == "true",
+ ReturnBody: r.FormValue("return_body") == "true",
  }
  if err := h.New(p, res); err != nil {
  log.Infof("error initializing dataset: %s", err.Error())
  util.WriteErrResponse(w, http.StatusInternalServerError, err)
  return
  }
+ if p.ReturnBody {
+ // TODO - this'll only work for JSON responses
+ data := []byte{}
+ if err := json.NewDecoder(res.Dataset.Body.(io.Reader)).Decode(&data); err != nil {
+ util.WriteErrResponse(w, http.StatusInternalServerError, err)
+ return
+ }
+ res.Dataset.Body = json.RawMessage(data)
+ }
  util.WriteResponse(w, res.Dataset)
 }
 

cmd/new.go

@@ -65,6 +65,7 @@ create a dataset with a dataset data file:
  cmd.Flags().BoolVarP(&o.Private, "private", "", false, "make dataset private. WARNING: not yet implimented. Please refer to https://github.com/qri-io/qri/issues/291 for updates")
  cmd.Flags().StringSliceVar(&o.Secrets, "secrets", nil, "transform secrets as comma separated key,value,key,value,... sequence")
  cmd.Flags().BoolVarP(&o.Publish, "publish", "p", false, "publish this dataset to the registry")
+ cmd.Flags().BoolVar(&o.DryRun, "dry-run", false, "simulate creating a dataset")
 
  return cmd
 }
@@ -79,6 +80,7 @@ type NewOptions struct {
  Message string
  Private bool
  Publish bool
+ DryRun bool
  Secrets []string
 
  DatasetRequests *lib.DatasetRequests
@@ -187,6 +189,7 @@ func (o *NewOptions) Run(args []string) (err error) {
  Dataset: dsp,
  Private: o.Private,
  Publish: o.Publish,
+ DryRun: o.DryRun,
  }
 
  ref = repo.DatasetRef{}

cmd/save.go

@@ -70,6 +70,7 @@ commit message and title to the save.`,
  // cmd.Flags().BoolVarP(&o.ShowValidation, "show-validation", "s", false, "display a list of validation errors upon adding")
  cmd.Flags().StringSliceVar(&o.Secrets, "secrets", nil, "transform secrets as comma separated key,value,key,value,... sequence")
  cmd.Flags().BoolVarP(&o.Publish, "publish", "p", false, "publish this dataset to the registry")
+ cmd.Flags().BoolVar(&o.DryRun, "dry-run", false, "simulate saving a dataset")
 
  return cmd
 }
@@ -87,6 +88,7 @@ type SaveOptions struct {
  Rescursive bool
  ShowValidation bool
  Publish bool
+ DryRun bool
  Secrets []string
 
  DatasetRequests *lib.DatasetRequests
@@ -180,6 +182,7 @@ continue?`, true) {
  Dataset: dsp,
  Private: false,
  Publish: o.Publish,
+ DryRun: o.DryRun,
  }
 
  res := &repo.DatasetRef{}

lib/datasets.go

@@ -86,14 +86,21 @@ func (r *DatasetRequests) Get(ref *repo.DatasetRef, res *repo.DatasetRef) error
 
 // SaveParams encapsulates arguments to Init & Save
 type SaveParams struct {
- Dataset *dataset.DatasetPod // dataset to create
- Private bool // option to make dataset private. private data is not currently implimented, see https://github.com/qri-io/qri/issues/291 for updates
- Publish bool
+ Dataset *dataset.DatasetPod // dataset to create
+ Private bool // option to make dataset private. private data is not currently implimented, see https://github.com/qri-io/qri/issues/291 for updates
+ Publish bool
+ DryRun bool
+ ReturnBody bool // if true, res.Dataset.Body will be a cafs.file of the body
 }
 
 // New creates a new qri dataset from a source of data
 func (r *DatasetRequests) New(p *SaveParams, res *repo.DatasetRef) (err error) {
  if r.cli != nil {
+ if p.ReturnBody {
+ // can't send an io.Reader interface over RPC
+ p.ReturnBody = false
+ log.Error("cannot return body bytes over RPC, disabling body return")
+ }
  return r.cli.Call("DatasetRequests.New", p, res)
  }
 
@@ -109,7 +116,8 @@ func (r *DatasetRequests) New(p *SaveParams, res *repo.DatasetRef) (err error) {
  defer bodyFile.Close()
  }
 
- *res, err = actions.CreateDataset(r.node, p.Dataset.Name, ds, bodyFile, secrets, true)
+ var body cafs.File
+ *res, body, err = actions.CreateDataset(r.node, p.Dataset.Name, ds, bodyFile, secrets, p.DryRun, true)
  if err != nil {
  log.Debugf("error creating dataset: %s\n", err.Error())
  return err
@@ -123,7 +131,11 @@ func (r *DatasetRequests) New(p *SaveParams, res *repo.DatasetRef) (err error) {
  }
  }
 
- return actions.ReadDataset(r.node.Repo, res)
+ if p.ReturnBody {
+ res.Dataset.Body = body
+ }
+
+ return nil
 }
 
 // Save adds a history entry, updating a dataset
@@ -137,6 +149,11 @@ func (r *DatasetRequests) New(p *SaveParams, res *repo.DatasetRef) (err error) {
 // but still use the hash to add to dataset.BodyPath
 func (r *DatasetRequests) Save(p *SaveParams, res *repo.DatasetRef) (err error) {
  if r.cli != nil {
+ if p.ReturnBody {
+ // can't send an io.Reader interface over RPC
+ p.ReturnBody = false
+ log.Error("cannot return body bytes over RPC, disabling body return")
+ }
  return r.cli.Call("DatasetRequests.Save", p, res)
  }
 
@@ -149,7 +166,7 @@ func (r *DatasetRequests) Save(p *SaveParams, res *repo.DatasetRef) (err error)
  return err
  }
 
- ref, err := actions.CreateDataset(r.node, p.Dataset.Name, ds, body, secrets, true)
+ ref, body, err := actions.CreateDataset(r.node, p.Dataset.Name, ds, body, secrets, p.DryRun, true)
  if err != nil {
  log.Debugf("create ds error: %s\n", err.Error())
  return err
@@ -163,6 +180,10 @@ func (r *DatasetRequests) Save(p *SaveParams, res *repo.DatasetRef) (err error)
  }
  }
 
+ if p.ReturnBody {
+ res.Dataset.Body = body
+ }
+
  *res = ref
  return nil
 }

lib/datasets_test.go

@@ -92,6 +92,10 @@ func TestDatasetRequestsInit(t *testing.T) {
  {&dataset.DatasetPod{
  Name: "foo",
  Meta: &dataset.Meta{Title: "foo"},
+ Structure: &dataset.StructurePod{
+ Format: "json",
+ Schema: map[string]interface{}{"type": "array"},
+ },
  Transform: &dataset.TransformPod{
  ScriptPath: "testdata/tf/transform.sky",
  }},
@@ -109,7 +113,7 @@ func TestDatasetRequestsInit(t *testing.T) {
  Qri: "tf:0",
  Syntax: "skylark",
  SyntaxVersion: skytf.Version,
- ScriptPath: "/map/QmcjVAiafyztY4rKjmZQZQybMMEo9EPekSErzs6eHtadfg",
+ ScriptPath: "/map/QmQDnmqNrxhggPSKrx3DMxhXf4NDhe3SEQKr7egFMu23L6",
  },
  Structure: &dataset.StructurePod{
  Qri: "st:0",

lib/testdata/tf/transform.sky

@@ -1,5 +1,5 @@
 
 
-def transform(qri):
- qri.set_meta("title", "bar")
- return ["hello", "world"]
+def transform(ds):
+ ds.set_body(["hello", "world"])
+ return ds