fix: lots of nitty-gritty fixes in time for demo.

this commit adds in some work-in-progress stuff on repo graphing,
but it's mainly just a bunch of cleanup in time for a demo.

api/handlers/datasets.go

@@ -135,7 +135,7 @@ func (h *DatasetHandlers) getDatasetHandler(w http.ResponseWriter, r *http.Reque
  util.WriteErrResponse(w, http.StatusInternalServerError, err)
  return
  }
- util.WriteResponse(w, res.Dataset)
+ util.WriteResponse(w, res)
 }
 
 func (h *DatasetHandlers) initDatasetHandler(w http.ResponseWriter, r *http.Request) {

core/datasets.go

@@ -165,6 +165,13 @@ func (r *DatasetRequests) InitDataset(p *InitDatasetParams, res *repo.DatasetRef
  }
 
  ds := &dataset.Dataset{}
+ if p.Url != "" {
+ ds.DownloadUrl = p.Url
+ // if we're adding from a dataset url, set a default accrual periodicity of once a week
+ // this'll set us up to re-check urls over time
+ // TODO - make this configurable via a param
+ ds.AccrualPeriodicity = "R/P1W"
+ }
  if p.Metadata != nil {
  if err := json.NewDecoder(p.Metadata).Decode(ds); err != nil {
  return fmt.Errorf("error parsing metadata json: %s", err.Error())

repo/graph.go

@@ -0,0 +1,128 @@
+package repo
+
+// import (
+// "fmt"
+// "github.com/qri-io/dataset/dsfs"
+// "github.com/qri-io/dataset/dsgraph"
+// )
+
+// var walkParallelism = 4
+
+// func RepoGraph(r Repo) (*dsgraph.Node, error) {
+// root := &dsgraph.Node{Type: dsgraph.NtNamespace, Path: "root"}
+// err := WalkRepoDatasets(r, func(prev *dsgraph.Node) func(int, *DatasetRef, error) (bool, error) {
+// return func(depth int, ref *DatasetRef, e error) (kontinue bool, err error) {
+// if e != nil {
+// return false, e
+// }
+
+// ds := NodesFromDatasetRef(ref)
+// if depth == 0 {
+// prev.AddLinks(dsgraph.Link{Type: dsgraph.LtNamespaceTip, From: prev, To: ds})
+// } else {
+// prev.AddLinks(dsgraph.Link{Type: dsgraph.LtPrevious, From: prev, To: ds})
+// }
+// prev = ds
+// return true, nil
+// }
+// }(root))
+// return root, err
+// }
+
+// func NodesFromDatasetRef(ref *DatasetRef) *dsgraph.Node {
+// root := &dsgraph.Node{Type: dsgraph.NtDataset, Path: ref.Path.String()}
+// ds := ref.Dataset
+// if ds == nil {
+// return root
+// }
+
+// data := &dsgraph.Node{Type: dsgraph.NtData, Path: ds.Data.Path().String()}
+// prev := &dsgraph.Node{Type: dsgraph.NtDataset, Path: ds.Previous.Path().String()}
+// root.AddLinks(
+// dsgraph.Link{Type: dsgraph.LtDsData, From: root, To: data},
+// dsgraph.Link{Type: dsgraph.LtPrevious, From: root, To: prev},
+// )
+// // if ds.Commit.Path().String() != "" {
+// // commit := &dsgraph.Node{Type: dsgraph.NtCommit, Path: ds.Commit.Path()}
+// // root.AddLinks(dsgraph.Link{Type: dsgraph.LtDsData, From: root, To: data})
+// // }
+// if ds.AbstractStructure != nil && ds.AbstractStructure.Path().String() != "" {
+// abst := &dsgraph.Node{Type: dsgraph.NtAbstStructure, Path: ds.AbstractStructure.Path().String()}
+// root.AddLinks(dsgraph.Link{Type: dsgraph.LtAbstStructure, From: root, To: abst})
+// }
+// if ds.Query != nil && ds.Query.Path().String() != "" {
+// query := &dsgraph.Node{Type: dsgraph.NtQuery, Path: ds.Query.Path().String()}
+// root.AddLinks(dsgraph.Link{Type: dsgraph.LtQuery, From: root, To: query})
+// }
+
+// return root
+// }
+
+// // WalkDatasets visits every dataset in the history of a user's namespace
+// // Yes, this potentially a very expensive function to call, use sparingly.
+// func WalkRepoDatasets(r Repo, visit func(logdepth int, ref *DatasetRef, err error) (bool, error)) error {
+// store := r.Store()
+// count, err := r.NameCount()
+// if err != nil {
+// return err
+// } else if count == 0 {
+// return ErrRepoEmpty
+// }
+
+// if count < walkParallelism {
+// walkParallelism = count
+// }
+
+// doSection := func(idx, pageSize int, done chan error) {
+// refs, err := r.Namespace(pageSize, idx*pageSize)
+// if err != nil {
+// done <- err
+// return
+// }
+
+// for _, ref := range refs {
+// fmt.Println(ref.Path.String())
+// ref.Dataset, err = dsfs.LoadDatasetRefs(store, ref.Path)
+// kontinue, err := visit(0, ref, err)
+// if err != nil {
+// fmt.Println("top", err.Error())
+// done <- err
+// return
+// }
+// if !kontinue {
+// break
+// }
+
+// depth := 1
+// for ref.Dataset != nil && ref.Dataset.Previous.String() != "" && ref.Dataset.Previous.String() != "/" {
+// ref.Path = ref.Dataset.Previous
+// ref.Dataset, err = dsfs.LoadDatasetRefs(store, ref.Path)
+// kontinue, err = visit(depth, ref, err)
+// if err != nil {
+// fmt.Println("prev", err.Error())
+// done <- err
+// return
+// }
+// if !kontinue {
+// break
+// }
+// depth++
+// }
+// }
+// }
+
+// pageSize := count / walkParallelism
+// done := make(chan error, 0)
+// for i := 0; i < walkParallelism; i++ {
+// go doSection(i, pageSize, done)
+// }
+
+// for i := 0; i < walkParallelism; i++ {
+// err := <-done
+// if err != nil {
+// return err
+// }
+// }
+
+// return nil
+// }

repo/graph_test.go

@@ -0,0 +1,67 @@
+package repo
+
+// import (
+// "encoding/json"
+// "fmt"
+// "testing"
+
+// "github.com/ipfs/go-datastore"
+// "github.com/qri-io/cafs/memfs"
+// "github.com/qri-io/dataset"
+// "github.com/qri-io/qri/repo/profile"
+// )
+
+// var (
+// ds1 = &dataset.Dataset{
+// Previous: datastore.NewKey(""),
+// }
+// ds2 = &dataset.Dataset{
+// Previous: datastore.NewKey(""),
+// }
+// )
+
+// func TestRepoGraph(t *testing.T) {
+// store := memfs.NewMapstore()
+// p := &profile.Profile{}
+
+// r, err := NewMemRepo(p, store, nil, nil)
+// if err != nil {
+// t.Errorf("error creating test repo: %s", err.Error())
+// return
+// }
+
+// data1p, _ := store.Put(memfs.NewMemfileBytes("data1", []byte("dataset_1")), true)
+// ds1.Data = data1p
+// ds1j, _ := ds1.MarshalJSON()
+// ds1p, err := store.Put(memfs.NewMemfileBytes("ds1", ds1j), true)
+// if err != nil {
+// t.Errorf("error putting dataset: %s", err.Error())
+// return
+// }
+// r.PutDataset(ds1p, ds1)
+// r.PutName("ds1", ds1p)
+
+// data2p, _ := store.Put(memfs.NewMemfileBytes("data1", []byte("dataset_2")), true)
+// ds2.Data = data2p
+// ds2j, _ := ds1.MarshalJSON()
+// ds2p, err := store.Put(memfs.NewMemfileBytes("ds2", ds2j), true)
+// if err != nil {
+// t.Errorf("error putting dataset: %s", err.Error())
+// return
+// }
+// r.PutDataset(ds2p, ds2)
+// r.PutName("ds1", ds2p)
+
+// node, err := RepoGraph(r)
+// if err != nil {
+// t.Errorf("error generating repo graph: %s", err.Error())
+// return
+// }
+
+// data, err := json.Marshal(node)
+// if err != nil {
+// t.Errorf("json marshal error: %s", err.Error())
+// return
+// }
+// fmt.Println(data)
+// }

repo/repo.go

@@ -21,6 +21,8 @@ var (
  ErrNotFound = fmt.Errorf("repo: not found")
  // when a Namestore name is already taken
  ErrNameTaken = fmt.Errorf("repo: name already in use")
+ // when the repo has no datasets
+ ErrRepoEmpty = fmt.Errorf("repo: this repo contains no datasets")
 )
 
 // Repo is the interface for working with a qri repository