Skip to content

Commit bc09dcf

Browse files
b5b5
committed
fix: lots of nitty-gritty fixes in time for demo.
this commit adds in some work-in-progress stuff on repo graphing, but it's mainly just a bunch of cleanup in time for a demo.
1 parent 6ca1839 commit bc09dcf

File tree

11 files changed

+367
-34
lines changed

11 files changed

+367
-34
lines changed

‎api/handlers/datasets.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ func (h *DatasetHandlers) getDatasetHandler(w http.ResponseWriter, r *http.Reque
135135
util.WriteErrResponse(w, http.StatusInternalServerError, err)
136136
return
137137
}
138-
util.WriteResponse(w, res.Dataset)
138+
util.WriteResponse(w, res)
139139
}
140140

141141
func (h *DatasetHandlers) initDatasetHandler(w http.ResponseWriter, r *http.Request) {

‎core/datasets.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,13 @@ func (r *DatasetRequests) InitDataset(p *InitDatasetParams, res *repo.DatasetRef
165165
}
166166

167167
ds := &dataset.Dataset{}
168+
if p.Url != "" {
169+
ds.DownloadUrl = p.Url
170+
// if we're adding from a dataset url, set a default accrual periodicity of once a week
171+
// this'll set us up to re-check urls over time
172+
// TODO - make this configurable via a param
173+
ds.AccrualPeriodicity = "R/P1W"
174+
}
168175
if p.Metadata != nil {
169176
if err := json.NewDecoder(p.Metadata).Decode(ds); err != nil {
170177
return fmt.Errorf("error parsing metadata json: %s", err.Error())

‎repo/graph.go

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
package repo
2+
3+
// import (
4+
// "fmt"
5+
// "github.com/qri-io/dataset/dsfs"
6+
// "github.com/qri-io/dataset/dsgraph"
7+
// )
8+
9+
// var walkParallelism = 4
10+
11+
// func RepoGraph(r Repo) (*dsgraph.Node, error) {
12+
// root := &dsgraph.Node{Type: dsgraph.NtNamespace, Path: "root"}
13+
// err := WalkRepoDatasets(r, func(prev *dsgraph.Node) func(int, *DatasetRef, error) (bool, error) {
14+
// return func(depth int, ref *DatasetRef, e error) (kontinue bool, err error) {
15+
// if e != nil {
16+
// return false, e
17+
// }
18+
19+
// ds := NodesFromDatasetRef(ref)
20+
// if depth == 0 {
21+
// prev.AddLinks(dsgraph.Link{Type: dsgraph.LtNamespaceTip, From: prev, To: ds})
22+
// } else {
23+
// prev.AddLinks(dsgraph.Link{Type: dsgraph.LtPrevious, From: prev, To: ds})
24+
// }
25+
// prev = ds
26+
// return true, nil
27+
// }
28+
// }(root))
29+
// return root, err
30+
// }
31+
32+
// func NodesFromDatasetRef(ref *DatasetRef) *dsgraph.Node {
33+
// root := &dsgraph.Node{Type: dsgraph.NtDataset, Path: ref.Path.String()}
34+
// ds := ref.Dataset
35+
// if ds == nil {
36+
// return root
37+
// }
38+
39+
// data := &dsgraph.Node{Type: dsgraph.NtData, Path: ds.Data.Path().String()}
40+
// prev := &dsgraph.Node{Type: dsgraph.NtDataset, Path: ds.Previous.Path().String()}
41+
// root.AddLinks(
42+
// dsgraph.Link{Type: dsgraph.LtDsData, From: root, To: data},
43+
// dsgraph.Link{Type: dsgraph.LtPrevious, From: root, To: prev},
44+
// )
45+
// // if ds.Commit.Path().String() != "" {
46+
// // commit := &dsgraph.Node{Type: dsgraph.NtCommit, Path: ds.Commit.Path()}
47+
// // root.AddLinks(dsgraph.Link{Type: dsgraph.LtDsData, From: root, To: data})
48+
// // }
49+
// if ds.AbstractStructure != nil && ds.AbstractStructure.Path().String() != "" {
50+
// abst := &dsgraph.Node{Type: dsgraph.NtAbstStructure, Path: ds.AbstractStructure.Path().String()}
51+
// root.AddLinks(dsgraph.Link{Type: dsgraph.LtAbstStructure, From: root, To: abst})
52+
// }
53+
// if ds.Query != nil && ds.Query.Path().String() != "" {
54+
// query := &dsgraph.Node{Type: dsgraph.NtQuery, Path: ds.Query.Path().String()}
55+
// root.AddLinks(dsgraph.Link{Type: dsgraph.LtQuery, From: root, To: query})
56+
// }
57+
58+
// return root
59+
// }
60+
61+
// // WalkDatasets visits every dataset in the history of a user's namespace
62+
// // Yes, this potentially a very expensive function to call, use sparingly.
63+
// func WalkRepoDatasets(r Repo, visit func(logdepth int, ref *DatasetRef, err error) (bool, error)) error {
64+
// store := r.Store()
65+
// count, err := r.NameCount()
66+
// if err != nil {
67+
// return err
68+
// } else if count == 0 {
69+
// return ErrRepoEmpty
70+
// }
71+
72+
// if count < walkParallelism {
73+
// walkParallelism = count
74+
// }
75+
76+
// doSection := func(idx, pageSize int, done chan error) {
77+
// refs, err := r.Namespace(pageSize, idx*pageSize)
78+
// if err != nil {
79+
// done <- err
80+
// return
81+
// }
82+
83+
// for _, ref := range refs {
84+
// fmt.Println(ref.Path.String())
85+
// ref.Dataset, err = dsfs.LoadDatasetRefs(store, ref.Path)
86+
// kontinue, err := visit(0, ref, err)
87+
// if err != nil {
88+
// fmt.Println("top", err.Error())
89+
// done <- err
90+
// return
91+
// }
92+
// if !kontinue {
93+
// break
94+
// }
95+
96+
// depth := 1
97+
// for ref.Dataset != nil && ref.Dataset.Previous.String() != "" && ref.Dataset.Previous.String() != "/" {
98+
// ref.Path = ref.Dataset.Previous
99+
// ref.Dataset, err = dsfs.LoadDatasetRefs(store, ref.Path)
100+
// kontinue, err = visit(depth, ref, err)
101+
// if err != nil {
102+
// fmt.Println("prev", err.Error())
103+
// done <- err
104+
// return
105+
// }
106+
// if !kontinue {
107+
// break
108+
// }
109+
// depth++
110+
// }
111+
// }
112+
// }
113+
114+
// pageSize := count / walkParallelism
115+
// done := make(chan error, 0)
116+
// for i := 0; i < walkParallelism; i++ {
117+
// go doSection(i, pageSize, done)
118+
// }
119+
120+
// for i := 0; i < walkParallelism; i++ {
121+
// err := <-done
122+
// if err != nil {
123+
// return err
124+
// }
125+
// }
126+
127+
// return nil
128+
// }

‎repo/graph_test.go

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
package repo
2+
3+
// import (
4+
// "encoding/json"
5+
// "fmt"
6+
// "testing"
7+
8+
// "github.com/ipfs/go-datastore"
9+
// "github.com/qri-io/cafs/memfs"
10+
// "github.com/qri-io/dataset"
11+
// "github.com/qri-io/qri/repo/profile"
12+
// )
13+
14+
// var (
15+
// ds1 = &dataset.Dataset{
16+
// Previous: datastore.NewKey(""),
17+
// }
18+
// ds2 = &dataset.Dataset{
19+
// Previous: datastore.NewKey(""),
20+
// }
21+
// )
22+
23+
// func TestRepoGraph(t *testing.T) {
24+
// store := memfs.NewMapstore()
25+
// p := &profile.Profile{}
26+
27+
// r, err := NewMemRepo(p, store, nil, nil)
28+
// if err != nil {
29+
// t.Errorf("error creating test repo: %s", err.Error())
30+
// return
31+
// }
32+
33+
// data1p, _ := store.Put(memfs.NewMemfileBytes("data1", []byte("dataset_1")), true)
34+
// ds1.Data = data1p
35+
// ds1j, _ := ds1.MarshalJSON()
36+
// ds1p, err := store.Put(memfs.NewMemfileBytes("ds1", ds1j), true)
37+
// if err != nil {
38+
// t.Errorf("error putting dataset: %s", err.Error())
39+
// return
40+
// }
41+
// r.PutDataset(ds1p, ds1)
42+
// r.PutName("ds1", ds1p)
43+
44+
// data2p, _ := store.Put(memfs.NewMemfileBytes("data1", []byte("dataset_2")), true)
45+
// ds2.Data = data2p
46+
// ds2j, _ := ds1.MarshalJSON()
47+
// ds2p, err := store.Put(memfs.NewMemfileBytes("ds2", ds2j), true)
48+
// if err != nil {
49+
// t.Errorf("error putting dataset: %s", err.Error())
50+
// return
51+
// }
52+
// r.PutDataset(ds2p, ds2)
53+
// r.PutName("ds1", ds2p)
54+
55+
// node, err := RepoGraph(r)
56+
// if err != nil {
57+
// t.Errorf("error generating repo graph: %s", err.Error())
58+
// return
59+
// }
60+
61+
// data, err := json.Marshal(node)
62+
// if err != nil {
63+
// t.Errorf("json marshal error: %s", err.Error())
64+
// return
65+
// }
66+
// fmt.Println(data)
67+
// }

‎repo/repo.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ var (
2121
ErrNotFound = fmt.Errorf("repo: not found")
2222
// when a Namestore name is already taken
2323
ErrNameTaken = fmt.Errorf("repo: name already in use")
24+
// when the repo has no datasets
25+
ErrRepoEmpty = fmt.Errorf("repo: this repo contains no datasets")
2426
)
2527

2628
// Repo is the interface for working with a qri repository

‎vendor/github.com/qri-io/dataset/dataset.go

Lines changed: 12 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/qri-io/dataset/dsfs/dataset.go

Lines changed: 26 additions & 13 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/github.com/qri-io/dataset/dsfs/dataset_test.go

Lines changed: 14 additions & 14 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)