feat: working on remote dataset retrieval

.circleci/config.yml

@@ -12,7 +12,7 @@ jobs:
     steps:
       - checkout
       - run: mkdir -p $TEST_RESULTS
-      - run: go get github.com/jstemmer/go-junit-report github.com/golang/lint/golint
+      - run: go get github.com/jstemmer/go-junit-report github.com/golang/lint/golint github.com/qri-io/datasetDiffer
       - run: 
           name: Run Lint Tests
           command: golint ./...

api/handlers/datasets.go

@@ -264,21 +264,21 @@ func (h *DatasetHandlers) getStructuredDataHandler(w http.ResponseWriter, r *htt
 }
 
 func (h *DatasetHandlers) addHandler(w http.ResponseWriter, r *http.Request) {
-	p := &core.AddParams{}
+	p := &repo.DatasetRef{}
 	if r.Header.Get("Content-Type") == "application/json" {
 		if err := json.NewDecoder(r.Body).Decode(p); err != nil {
 			util.WriteErrResponse(w, http.StatusBadRequest, err)
 			return
 		}
 		// TODO - clean this up
-		p.Hash = r.URL.Path[len("/add/"):]
+		p.Path = r.URL.Path[len("/add/"):]
 		if p.Name == "" && r.FormValue("name") != "" {
 			p.Name = r.FormValue("name")
 		}
 	} else {
-		p = &core.AddParams{
+		p = &repo.DatasetRef{
 			Name: r.URL.Query().Get("name"),
-			Hash: r.URL.Path[len("/add/"):],
+			Path: r.URL.Path[len("/add/"):],
 		}
 	}
 

cmd/add.go

@@ -23,9 +23,8 @@ var datasetAddCmd = &cobra.Command{
 	Short:      "add a dataset to your local repository",
 	SuggestFor: []string{"init"},
 	Long: `
-Add creates a new dataset from data you supply. You can supply data from a file 
-or a URL. Please note that all data added to qri is made public on the 
-distributed web when you run qri connect.
+Add creates a new dataset from data you supply. Please note that all data added 
+to qri is made public on the distributed web when you run qri connect.
 
 When adding data, you can supply metadata and dataset structure, but it’s not 
 required. qri does what it can to infer the details you don’t provide. 
@@ -47,15 +46,11 @@ changes to qri.`,
 				ref, err := repo.ParseDatasetRef(arg)
 				ExitIfErr(err)
 
-				req, err := datasetRequests(false)
+				req, err := datasetRequests(true)
 				ExitIfErr(err)
 
-				p := &core.AddParams{
-					Name: ref.Name,
-					Hash: ref.Path,
-				}
 				res := &repo.DatasetRef{}
-				err = req.Add(p, res)
+				err = req.Add(ref, res)
 				ExitIfErr(err)
 				printInfo("Successfully added dataset %s: %s", addDsName, res.Path)
 			}

cmd/cmd_test.go

@@ -98,7 +98,7 @@ func TestCommandsIntegration(t *testing.T) {
 		{"info"},
 		{"add", "-f" + moviesFilePath, "-n" + "movies"},
 		{"list"},
-		{"save", "--data=" + movies2FilePath, "-m" + "commit_1", "me/movies"},
+		{"save", "--data=" + movies2FilePath, "-t" + "commit_1", "me/movies"},
 		{"log", "me/movies"},
 		{"export", "--dataset", "me/movies", "-o" + path},
 		{"rename", "me/movies", "me/movie"},

cmd/config.go

@@ -179,8 +179,8 @@ type Config struct {
 	IPFSPath string
 	// Datastore configuration details
 	// Datastore       DatastoreCfg
-	// DefaultDatasets is a list of datasets to grab on initially joining the network
-	DefaultDatasets map[string]string
+	// DefaultDatasets is a list of dataset references to grab on initially joining the network
+	DefaultDatasets []string
 }
 
 // TODO - Is this is the right place for this?
@@ -268,9 +268,9 @@ func defaultCfgBytes() []byte {
 		Bootstrap:   p2p.DefaultBootstrapAddresses,
 		// defaultDatasets is a hard-coded dataset added when a new qri repo is created
 		// these hashes should always/highly available
-		DefaultDatasets: map[string]string{
+		DefaultDatasets: []string{
 			// fivethirtyeight comic characters
-			"comic_characters": "/ipfs/QmcqkHFA2LujZxY38dYZKmxsUstN4unk95azBjwEhwrnM6/dataset.json",
+			"me/comic_characters@/ipfs/QmcqkHFA2LujZxY38dYZKmxsUstN4unk95azBjwEhwrnM6/dataset.json",
 		},
 	}
 

cmd/connect.go

@@ -124,18 +124,20 @@ func initializeDistributedAssets(node *p2p.QriNode) {
 
 	req := core.NewDatasetRequests(node.Repo, nil)
 
-	for name, path := range cfg.DefaultDatasets {
-		fmt.Printf("attempting to add default dataset: %s\n", path)
+	for _, refstr := range cfg.DefaultDatasets {
+		fmt.Printf("attempting to add default dataset: %s\n", refstr)
+		ref, err := repo.ParseDatasetRef(refstr)
+		if err != nil {
+			fmt.Println("error parsing dataset reference: '%s': %s", refstr, err.Error())
+			continue
+		}
 		res := &repo.DatasetRef{}
-		err := req.Add(&core.AddParams{
-			Hash: path,
-			Name: name,
-		}, res)
+		err = req.Add(ref, res)
 		if err != nil {
-			fmt.Printf("add dataset %s error: %s\n", path, err.Error())
+			fmt.Printf("add dataset %s error: %s\n", refstr, err.Error())
 			return
 		}
-		fmt.Printf("added default dataset: %s\n", path)
+		fmt.Printf("added default dataset: %s\n", refstr)
 	}
 
 	cfg.Initialized = true

cmd/repo.go

@@ -13,11 +13,14 @@ import (
 	"github.com/qri-io/qri/repo/fs"
 )
 
-var r repo.Repo
+var (
+	repository repo.Repo
+	rpcClient  *rpc.Client
+)
 
 func getRepo(online bool) repo.Repo {
-	if r != nil {
-		return r
+	if repository != nil {
+		return repository
 	}
 
 	if !QRIRepoInitialized() {
@@ -114,6 +117,12 @@ func peerRequests(online bool) (*core.PeerRequests, error) {
 }
 
 func repoOrClient(online bool) (repo.Repo, *rpc.Client, error) {
+	if repository != nil {
+		return repository, nil, nil
+	} else if rpcClient != nil {
+		return nil, rpcClient, nil
+	}
+
 	if fs, err := ipfs.NewFilestore(func(cfg *ipfs.StoreCfg) {
 		cfg.FsRepoPath = IpfsFsPath
 		cfg.Online = online
@@ -185,11 +194,14 @@ func qriNode(online bool) (node *p2p.QriNode, err error) {
 		return
 	}
 
+	// if online {
+	// 	log.Info("p2p addresses:")
+	// 	for _, a := range node.EncapsulatedAddresses() {
+	// 		log.Infof("  %s", a.String())
+	// 	}
+	// }
 	if online {
-		log.Info("p2p addresses:")
-		for _, a := range node.EncapsulatedAddresses() {
-			log.Infof("  %s", a.String())
-		}
+		log.Infof("connecting to the distributed web...")
 	}
 
 	return

cmd/save.go

@@ -47,23 +47,22 @@ collaboration are in the works. Sit tight sportsfans.`,
 			ErrExit(fmt.Errorf("please provide the name of an existing dataset so save updates to"))
 		}
 		if saveMetaFile == "" && saveDataFile == "" && saveStructureFile == "" {
-			ErrExit(fmt.Errorf("either a metadata or data option is required"))
+			ErrExit(fmt.Errorf("one of --structure, --meta or --data is required"))
 		}
 
 		ref, err := repo.ParseDatasetRef(args[0])
 		ExitIfErr(err)
 
-		req, err := datasetRequests(false)
-		ExitIfErr(err)
+		// req, err := datasetRequests(false)
+		// ExitIfErr(err)
 
-		// TODO - need to make sure users aren't forking by referncing commits other than tip
-		p := &repo.DatasetRef{
-			Name: ref.Name,
-			Path: ref.Path,
-		}
+		// TODO - this is silly:
+		ref.Peername = ""
+
+		req := core.NewDatasetRequests(getRepo(false), nil)
 
 		prev := &repo.DatasetRef{}
-		err = req.Get(p, prev)
+		err = req.Get(ref, prev)
 		ExitIfErr(err)
 
 		save := &core.SaveParams{}

core/datasets.go

@@ -35,6 +35,13 @@ type DatasetRequests struct {
 	Node *p2p.QriNode
 }
 
+// Repo exposes the DatasetRequest's repo
+// TODO - this is an architectural flaw resulting from not having a clear
+// order of local > network > RPC requests figured out
+func (r *DatasetRequests) Repo() repo.Repo {
+	return r.repo
+}
+
 // CoreRequestsName implements the Requets interface
 func (DatasetRequests) CoreRequestsName() string { return "datasets" }
 
@@ -61,7 +68,7 @@ func (r *DatasetRequests) List(p *ListParams, res *[]*repo.DatasetRef) error {
 		replies, err := r.Node.RequestDatasetsList(p.Peername)
 		*res = replies
 		return err
-	} else if r.Node == nil {
+	} else if p.Peername != "" && r.Node == nil {
 		return fmt.Errorf("cannot list datasets of peer without p2p connection")
 	}
 
@@ -108,8 +115,9 @@ func (r *DatasetRequests) Get(p *repo.DatasetRef, res *repo.DatasetRef) error {
 
 	getRemote := func(err error) error {
 		if r.Node != nil {
-			ds, err := r.Node.RequestDatasetInfo(p)
-			if ds != nil {
+			ref, err := r.Node.RequestDatasetInfo(p)
+			if ref != nil {
+				ds := ref.Dataset
 				st := ds.Structure
 
 				// TODO - it seems that jsonschema.RootSchema and encoding/gob
@@ -120,7 +128,7 @@ func (r *DatasetRequests) Get(p *repo.DatasetRef, res *repo.DatasetRef) error {
 				*res = repo.DatasetRef{
 					Peername: p.Peername,
 					Name:     p.Name,
-					Path:     ds.Path().String(),
+					Path:     ref.Path,
 					Dataset: &dataset.Dataset{
 						Commit: ds.Commit,
 						Meta:   ds.Meta,
@@ -132,6 +140,7 @@ func (r *DatasetRequests) Get(p *repo.DatasetRef, res *repo.DatasetRef) error {
 					},
 				}
 			}
+			fmt.Println("remote err:", err.Error())
 			return err
 		}
 		return err
@@ -159,9 +168,10 @@ func (r *DatasetRequests) Get(p *repo.DatasetRef, res *repo.DatasetRef) error {
 	}
 
 	*res = repo.DatasetRef{
-		Name:    name,
-		Path:    p.Path,
-		Dataset: ds,
+		Peername: p.Peername,
+		Name:     name,
+		Path:     ds.Path().String(),
+		Dataset:  ds,
 	}
 	return nil
 }
@@ -307,6 +317,7 @@ type SaveParams struct {
 }
 
 // Save adds a history entry, updating a dataset
+// TODO - need to make sure users aren't forking by referncing commits other than tip
 func (r *DatasetRequests) Save(p *SaveParams, res *repo.DatasetRef) (err error) {
 	if r.cli != nil {
 		return r.cli.Call("DatasetRequests.Save", p, res)
@@ -320,29 +331,26 @@ func (r *DatasetRequests) Save(p *SaveParams, res *repo.DatasetRef) (err error)
 
 	ds := &dataset.Dataset{}
 
-	rt, ref := dsfs.RefType(p.Changes.PreviousPath)
-	// allows using dataset names as "previous" fields
-	if rt == "name" {
-		name = ref
-		prevpath, err = r.repo.GetPath(strings.Trim(ref, "/"))
-		if err != nil {
-			return fmt.Errorf("error getting previous dataset path: %s", err.Error())
-		}
-	} else {
-		prevpath = datastore.NewKey(ref)
-		// attempt to grab name for later if path is provided
-		name, _ = r.repo.GetName(prevpath)
-	}
+	// read previous changes
+	// prev, err := r.repo.GetDataset(datastore.NewKey(p.Changes.PreviousPath))
+	// if err != nil {
+	// 	return fmt.Errorf("error getting previous dataset: %s", err.Error())
+	// }
 
 	// read previous changes
-	prev, err := r.repo.GetDataset(prevpath)
+	prev, err := dsfs.LoadDataset(r.repo.Store(), datastore.NewKey(p.Changes.PreviousPath))
 	if err != nil {
 		return fmt.Errorf("error getting previous dataset: %s", err.Error())
 	}
 
 	// add all previous fields and any changes
 	ds.Assign(prev, p.Changes)
 
+	if err := dsfs.DerefDataset(r.repo.Store(), prev); err != nil {
+		return fmt.Errorf("error dereferencing dataset: %s", err.Error())
+	}
+	fmt.Println("huh???", prev.Meta)
+
 	if strings.HasSuffix(prevpath.String(), dsfs.PackageFileDataset.String()) {
 		ds.PreviousPath = strings.TrimSuffix(prevpath.String(), "/"+dsfs.PackageFileDataset.String())
 	} else {
@@ -549,24 +557,28 @@ func (r *DatasetRequests) StructuredData(p *StructuredDataParams, data *Structur
 	return nil
 }
 
-// AddParams defines parameters for adding a dataset
-type AddParams struct {
-	Name string
-	Hash string
-}
-
 // Add adds an existing dataset to a peer's repository
-func (r *DatasetRequests) Add(p *AddParams, res *repo.DatasetRef) (err error) {
+func (r *DatasetRequests) Add(ref *repo.DatasetRef, res *repo.DatasetRef) (err error) {
 	if r.cli != nil {
-		return r.cli.Call("DatasetRequests.Add", p, res)
+		return r.cli.Call("DatasetRequests.Add", ref, res)
+	}
+
+	if ref.Path == "" && r.Node != nil {
+		res, err := r.Node.RequestDatasetInfo(ref)
+		if err != nil {
+			return err
+		}
+		fmt.Println(res)
+		ref = res
+		fmt.Println(ref.Path)
 	}
 
 	fs, ok := r.repo.Store().(*ipfs.Filestore)
 	if !ok {
 		return fmt.Errorf("can only add datasets when running an IPFS filestore")
 	}
 
-	key := datastore.NewKey(strings.TrimSuffix(p.Hash, "/"+dsfs.PackageFileDataset.String()))
+	key := datastore.NewKey(strings.TrimSuffix(ref.Path, "/"+dsfs.PackageFileDataset.String()))
 	_, err = fs.Fetch(cafs.SourceAny, key)
 	if err != nil {
 		return fmt.Errorf("error fetching file: %s", err.Error())
@@ -578,7 +590,7 @@ func (r *DatasetRequests) Add(p *AddParams, res *repo.DatasetRef) (err error) {
 	}
 
 	path := datastore.NewKey(key.String() + "/" + dsfs.PackageFileDataset.String())
-	err = r.repo.PutName(p.Name, path)
+	err = r.repo.PutName(ref.Name, path)
 	if err != nil {
 		return fmt.Errorf("error putting dataset name in repo: %s", err.Error())
 	}
@@ -589,7 +601,7 @@ func (r *DatasetRequests) Add(p *AddParams, res *repo.DatasetRef) (err error) {
 	}
 
 	*res = repo.DatasetRef{
-		Name:    p.Name,
+		Name:    ref.Name,
 		Path:    path.String(),
 		Dataset: ds,
 	}

core/datasets_test.go

@@ -349,11 +349,11 @@ func TestDatasetRequestsStructuredData(t *testing.T) {
 
 func TestDatasetRequestsAdd(t *testing.T) {
 	cases := []struct {
-		p   *AddParams
+		p   *repo.DatasetRef
 		res *repo.DatasetRef
 		err string
 	}{
-		{&AddParams{Name: "abc", Hash: "hash###"}, nil, "can only add datasets when running an IPFS filestore"},
+		{&repo.DatasetRef{Name: "abc", Path: "hash###"}, nil, "can only add datasets when running an IPFS filestore"},
 	}
 
 	mr, err := testrepo.NewTestRepo()