feat(migrate): remove qri-only IPFS repo on config migration

cmd/cmd.go

@@ -51,8 +51,7 @@ func Execute() {
 	root.SilenceErrors = true
 	// Execute the subcommand
 	if err := root.Execute(); err != nil {
-		printErr(os.Stderr, err)
-		os.Exit(1)
+		ErrExit(os.Stderr, err)
 	}
 
 	<-shutdown()

config/migrate/migrate.go

@@ -2,21 +2,26 @@
 package migrate
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
+	logging "github.com/ipfs/go-log"
 	"github.com/mitchellh/go-homedir"
 	"github.com/qri-io/ioes"
 	"github.com/qri-io/qfs/qipfs"
 	"github.com/qri-io/qri/config"
 	qerr "github.com/qri-io/qri/errors"
+	"github.com/qri-io/qri/repo/buildrepo"
 )
 
 var (
+	log = logging.Logger("migrate")
 	// ErrNeedMigration indicates a migration is required
 	ErrNeedMigration = fmt.Errorf("migration required")
 	// ErrMigrationSucceeded indicates a migration completed executing
@@ -99,9 +104,10 @@ func ZeroToOne(cfg *config.Config) error {
 func OneToTwo(cfg *config.Config) error {
 	qriPath := filepath.Dir(cfg.Path())
 	newIPFSPath := filepath.Join(qriPath, "ipfs")
+	oldIPFSPath := configVersionOneIPFSPath()
 
 	// TODO(ramfox): qfs migration
-	if err := qipfs.InternalizeIPFSRepo(configVersionOneIPFSPath(), newIPFSPath); err != nil {
+	if err := qipfs.InternalizeIPFSRepo(oldIPFSPath, newIPFSPath); err != nil {
 		return err
 	}
 
@@ -117,7 +123,14 @@ func OneToTwo(cfg *config.Config) error {
 		return err
 	}
 
-	// TODO(ramfox): remove original ipfs repo after all migrations were successful
+	if err := maybeRemoveIPFSRepo(cfg, oldIPFSPath); err != nil {
+		log.Debug(err)
+		fmt.Printf("error removing IPFS repo at %q:\n\t%s", oldIPFSPath, err)
+		fmt.Printf(`qri has successfully internalized this IPFS repo, and no longer 
+		needs the folder at %q. you may want to remove it
+`, oldIPFSPath)
+	}
+
 	return nil
 }
 
@@ -219,3 +232,75 @@ func prompt(w io.Writer, r io.Reader, msg string) string {
 	fmt.Fscanln(r, &input)
 	return strings.TrimSpace(strings.ToLower(input))
 }
+
+func maybeRemoveIPFSRepo(cfg *config.Config, oldPath string) error {
+	fmt.Println("\nchecking if existing IPFS directory contains non-qri data...")
+	repoPath := filepath.Dir(cfg.Path())
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+
+	r, err := buildrepo.New(ctx, repoPath, cfg)
+	if err != nil {
+		return err
+	}
+
+	// Note: this is intentionally using the new post-migration IPFS repo to judge
+	// pin presence, because we can't operate on the old one
+	fs := r.Filesystem().Filesystem(qipfs.FilestoreType)
+	if fs == nil {
+		return nil
+	}
+
+	logbookPaths, err := r.Logbook().AllReferencedDatasetPaths(ctx)
+	if err != nil {
+		return err
+	}
+
+	paths := map[string]struct{}{
+		// add common paths auto-added on IPFS init we can safely ignore
+		"/ipld/QmQPeNsJPyVWPFDVHb77w8G42Fvo15z4bG2X8D2GhfbSXc": {},
+		"/ipld/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn": {},
+		"/ipld/QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv": {},
+	}
+
+	for p := range logbookPaths {
+		// switch "/ipfs/" prefixes for "/ipld/"
+		p = strings.Replace(p, "/ipfs", "/ipld", 1)
+		paths[p] = struct{}{}
+	}
+
+	unknownPinCh, err := fs.(*qipfs.Filestore).PinsetDifference(ctx, paths)
+	if err != nil {
+		return err
+	}
+
+	log.Debugf("checking pins...%#v\n", paths)
+
+	unknown := []string{}
+	for path := range unknownPinCh {
+		log.Debugf("checking if unknown pin is a dataset: %s\n", path)
+		path = strings.Replace(path, "/ipld", "/ipfs", 1)
+		// check if the pinned path is a valid qri dataset, looking for "dataset.json"
+		// this check allows us to ignore qri data logbook doesn't know about
+		if f, err := fs.Get(ctx, fmt.Sprintf("%s/dataset.json", path)); err == nil {
+			f.Close()
+		} else {
+			unknown = append(unknown, path)
+		}
+	}
+
+	if len(unknown) > 0 {
+		fmt.Printf(`qri left your original IPFS repo in place because it contains pinned data that 
+qri isn't managing. Qri has created an internal copy of your IPFS repo, and no
+longer requires the repo at %q`, oldPath)
+		if len(unknown) < 10 {
+			fmt.Printf("unknown pins:\n\t%s\n", strings.Join(unknown, "\n\t"))
+		} else {
+			fmt.Printf("found %d unknown pins\n", len(unknown))
+		}
+		return nil
+	}
+
+	fmt.Printf("moved IPFS repo from %q into qri repo\n", oldPath)
+	return os.RemoveAll(oldPath)
+}

go.mod

@@ -41,7 +41,7 @@ require (
 	github.com/qri-io/doggos v0.1.0
 	github.com/qri-io/ioes v0.1.1
 	github.com/qri-io/jsonschema v0.2.0
-	github.com/qri-io/qfs v0.1.1-0.20200623190921-7764b79b00d9
+	github.com/qri-io/qfs v0.1.1-0.20200624184248-011ef3915f0c
 	github.com/qri-io/starlib v0.4.2-0.20200406084526-0e4566b610a1
 	github.com/russross/blackfriday/v2 v2.0.2-0.20190629151518-3e56bb68c887
 	github.com/sergi/go-diff v1.0.0

go.sum

@@ -863,8 +863,8 @@ github.com/qri-io/jsonpointer v0.1.1/go.mod h1:DnJPaYgiKu56EuDp8TU5wFLdZIcAnb/uH
 github.com/qri-io/jsonschema v0.2.0 h1:is8lirh3HYwTkC0e+4jL/vWEHwzPLojnl4FWkUoeEPU=
 github.com/qri-io/jsonschema v0.2.0/go.mod h1:g7DPkiOsK1xv6T/Ao5scXRkd+yTFygcANPBaaqW+VrI=
 github.com/qri-io/qfs v0.1.1-0.20200623175200-23690284798a/go.mod h1:NP4Va1b0/V4nbyPdX3FmZ4cJTPatn0d5FZF5GBxcIJA=
-github.com/qri-io/qfs v0.1.1-0.20200623190921-7764b79b00d9 h1:Jot1vQPOrvMk9T7K61uLX9sXPdeQQK7eS4WQALTaxxA=
-github.com/qri-io/qfs v0.1.1-0.20200623190921-7764b79b00d9/go.mod h1:NP4Va1b0/V4nbyPdX3FmZ4cJTPatn0d5FZF5GBxcIJA=
+github.com/qri-io/qfs v0.1.1-0.20200624184248-011ef3915f0c h1:hei43xn1aGxkQs6pSeusTdlzlc/eZCzi2RtPRf+ABgQ=
+github.com/qri-io/qfs v0.1.1-0.20200624184248-011ef3915f0c/go.mod h1:NP4Va1b0/V4nbyPdX3FmZ4cJTPatn0d5FZF5GBxcIJA=
 github.com/qri-io/starlib v0.4.2-0.20200406084526-0e4566b610a1/go.mod h1:Nno0nW88BMydkiYjFYwswcQxQTSYhYAivSdFwlm5l04=
 github.com/qri-io/varName v0.1.0/go.mod h1:IGWuuGOHhLJ9ZZg28C/+oMYm1QYP+pAorNZKQpdXhxQ=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=

logbook/logbook.go

@@ -634,6 +634,43 @@ func (book Book) ListAllLogs(ctx context.Context) ([]*oplog.Log, error) {
 	return book.store.Logs(ctx, 0, -1)
 }
 
+// AllReferencedDatasetPaths scans an entire logbook looking for dataset paths
+func (book *Book) AllReferencedDatasetPaths(ctx context.Context) (map[string]struct{}, error) {
+	paths := map[string]struct{}{}
+	logs, err := book.ListAllLogs(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, l := range logs {
+		addReferencedPaths(l, paths)
+	}
+	return paths, nil
+}
+
+func addReferencedPaths(log *oplog.Log, paths map[string]struct{}) {
+	ps := []string{}
+	for _, op := range log.Ops {
+		if op.Model == CommitModel {
+			switch op.Type {
+			case oplog.OpTypeInit:
+				ps = append(ps, op.Ref)
+			case oplog.OpTypeRemove:
+				ps = ps[:len(ps)-int(op.Size)]
+			case oplog.OpTypeAmend:
+				ps[len(ps)-1] = op.Ref
+			}
+		}
+	}
+	for _, p := range ps {
+		paths[p] = struct{}{}
+	}
+
+	for _, l := range log.Logs {
+		addReferencedPaths(l, paths)
+	}
+}
+
 // Log gets a log for a given ID
 func (book Book) Log(ctx context.Context, id string) (*oplog.Log, error) {
 	return book.store.Get(ctx, id)

p2p/test/ipfs.go

@@ -42,7 +42,7 @@ func MakeRepoFromIPFSNode(ctx context.Context, node *core.IpfsNode, username str
 		return nil, err
 	}
 
-	ipfs, err := qipfs.NewFilesystemFromNode(node)
+	ipfs, err := qipfs.NewFilesystemFromNode(ctx, node)
 	if err != nil {
 		return nil, err
 	}