From a3bd3bc3641f9ee23d7a3b64058ef2cedc8d0e78 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 1 Jul 2016 16:10:19 -0700 Subject: [PATCH 1/7] Automatically download and run migrations if needed License: MIT Signed-off-by: Jeromy --- core/commands/repo.go | 2 +- core/commands/version.go | 2 +- core/corerepo/stat.go | 4 +- repo/fsrepo/fsrepo.go | 26 +++- repo/fsrepo/migrations/mfsr.go | 43 ++++-- repo/fsrepo/migrations/migrations.go | 220 +++++++++++++++++++++++++++ repo/fsrepo/migrations/unpack.go | 101 ++++++++++++ 7 files changed, 382 insertions(+), 16 deletions(-) create mode 100644 repo/fsrepo/migrations/migrations.go create mode 100644 repo/fsrepo/migrations/unpack.go diff --git a/core/commands/repo.go b/core/commands/repo.go index 9eada31250a..7e80c99a073 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -320,7 +320,7 @@ var repoVersionCmd = &cmds.Command{ }, Run: func(req cmds.Request, res cmds.Response) { res.SetOutput(&RepoVersion{ - Version: fsrepo.RepoVersion, + Version: fmt.Sprint(fsrepo.RepoVersion), }) }, Type: RepoVersion{}, diff --git a/core/commands/version.go b/core/commands/version.go index b53d36da78a..ab7761fa6ce 100644 --- a/core/commands/version.go +++ b/core/commands/version.go @@ -35,7 +35,7 @@ var VersionCmd = &cmds.Command{ res.SetOutput(&VersionOutput{ Version: config.CurrentVersionNumber, Commit: config.CurrentCommit, - Repo: fsrepo.RepoVersion, + Repo: fmt.Sprint(fsrepo.RepoVersion), System: runtime.GOARCH + "/" + runtime.GOOS, //TODO: Precise version here Golang: runtime.Version(), }) diff --git a/core/corerepo/stat.go b/core/corerepo/stat.go index 77dcaf6862e..a419041da03 100644 --- a/core/corerepo/stat.go +++ b/core/corerepo/stat.go @@ -1,6 +1,8 @@ package corerepo import ( + "fmt" + "github.com/ipfs/go-ipfs/core" fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" @@ -40,6 +42,6 @@ func RepoStat(n *core.IpfsNode, ctx context.Context) (*Stat, error) { NumObjects: count, RepoSize: usage, RepoPath: path, - Version: "fs-repo@" + fsrepo.RepoVersion, + Version: fmt.Sprintf("fs-repo@%d", fsrepo.RepoVersion), }, nil } diff --git a/repo/fsrepo/fsrepo.go b/repo/fsrepo/fsrepo.go index 3c369b498ad..e6ae0d384d0 100644 --- a/repo/fsrepo/fsrepo.go +++ b/repo/fsrepo/fsrepo.go @@ -26,7 +26,7 @@ import ( var log = logging.Logger("fsrepo") // version number that we are currently expecting to see -var RepoVersion = "4" +var RepoVersion = 4 var migrationInstructions = `See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md Sorry for the inconvenience. In the future, these will run automatically.` @@ -36,6 +36,12 @@ Program version is: %s Please run the ipfs migration tool before continuing. ` + migrationInstructions +var programTooLowMessage = `Your programs version (%d) is lower than your repos (%d). +Please update ipfs to a version that supports the existing repo, or run +a migration in reverse. + +See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md for details.` + var ( ErrNoVersion = errors.New("no version file found, please run 0-to-1 migration tool.\n" + migrationInstructions) ErrOldRepo = errors.New("ipfs repo found in old '~/.go-ipfs' location, please run migration tool.\n" + migrationInstructions) @@ -134,8 +140,22 @@ func open(repoPath string) (repo.Repo, error) { return nil, err } - if ver != RepoVersion { - return nil, fmt.Errorf(errIncorrectRepoFmt, ver, RepoVersion) + if RepoVersion > ver { + r.lockfile.Close() + + err := mfsr.TryMigrating(RepoVersion) + if err != nil { + return nil, err + } + + r.lockfile, err = lockfile.Lock(r.path) + if err != nil { + return nil, fmt.Errorf("reacquiring lock: %s", err) + } + + } else if ver > RepoVersion { + // program version too low for existing repo + return nil, fmt.Errorf(programTooLowMessage, RepoVersion, ver) } // check repo path, then check all constituent parts. diff --git a/repo/fsrepo/migrations/mfsr.go b/repo/fsrepo/migrations/mfsr.go index c591f67eef3..1370c183281 100644 --- a/repo/fsrepo/migrations/mfsr.go +++ b/repo/fsrepo/migrations/mfsr.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "os" "path" + "strconv" "strings" ) @@ -16,27 +17,26 @@ func (rp RepoPath) VersionFile() string { return path.Join(string(rp), VersionFile) } -func (rp RepoPath) Version() (string, error) { +func (rp RepoPath) Version() (int, error) { if rp == "" { - return "", fmt.Errorf("invalid repo path \"%s\"", rp) + return 0, fmt.Errorf("invalid repo path \"%s\"", rp) } fn := rp.VersionFile() if _, err := os.Stat(fn); os.IsNotExist(err) { - return "", VersionFileNotFound(rp) + return 0, VersionFileNotFound(rp) } c, err := ioutil.ReadFile(fn) if err != nil { - return "", err + return 0, err } - s := string(c) - s = strings.TrimSpace(s) - return s, nil + s := strings.TrimSpace(string(c)) + return strconv.Atoi(s) } -func (rp RepoPath) CheckVersion(version string) error { +func (rp RepoPath) CheckVersion(version int) error { v, err := rp.Version() if err != nil { return err @@ -49,9 +49,9 @@ func (rp RepoPath) CheckVersion(version string) error { return nil } -func (rp RepoPath) WriteVersion(version string) error { +func (rp RepoPath) WriteVersion(version int) error { fn := rp.VersionFile() - return ioutil.WriteFile(fn, []byte(version+"\n"), 0644) + return ioutil.WriteFile(fn, []byte(fmt.Sprintf("%d\n", version)), 0644) } type VersionFileNotFound string @@ -59,3 +59,26 @@ type VersionFileNotFound string func (v VersionFileNotFound) Error() string { return "no version file in repo at " + string(v) } + +func TryMigrating(tovers int) error { + if !YesNoPrompt("run migrations automatically? [y/n]") { + return fmt.Errorf("please run the migrations manually") + } + + return RunMigration(tovers) +} + +func YesNoPrompt(prompt string) bool { + var s string + for { + fmt.Printf("%s ", prompt) + fmt.Scanf("%s", &s) + switch s { + case "y", "Y": + return true + case "n", "N": + return false + } + fmt.Println("Please press either 'y' or 'n'") + } +} diff --git a/repo/fsrepo/migrations/migrations.go b/repo/fsrepo/migrations/migrations.go new file mode 100644 index 00000000000..b6558d510de --- /dev/null +++ b/repo/fsrepo/migrations/migrations.go @@ -0,0 +1,220 @@ +package mfsr + +import ( + "bufio" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" +) + +var DistPath = "https://ipfs.io/ipns/dist.ipfs.io" + +const migrations = "fs-repo-migrations" + +func RunMigration(newv int) error { + migrateBin := "fs-repo-migrations" + fmt.Println(" => checking for migrations binary...") + _, err := exec.LookPath(migrateBin) + if err == nil { + // check to make sure migrations binary supports our target version + err = verifyMigrationSupportsVersion(migrateBin, newv) + } + + if err != nil { + fmt.Println(" => usable migrations not found on system, fetching...") + loc, err := GetMigrations() + if err != nil { + return err + } + + err = verifyMigrationSupportsVersion(loc, newv) + if err != nil { + return fmt.Errorf("could not find migrations binary that supports version %d", newv) + } + + migrateBin = loc + } + + cmd := exec.Command(migrateBin, "-to", fmt.Sprint(newv), "-y") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + fmt.Printf(" => running migration: '%s -to %d -y'\n\n", migrateBin, newv) + + err = cmd.Run() + if err != nil { + return fmt.Errorf("migration failed: %s", err) + } + + fmt.Println(" => migrations binary completed successfully") + + return nil +} + +func GetMigrations() (string, error) { + latest, err := GetLatestVersion(DistPath, migrations) + if err != nil { + return "", fmt.Errorf("getting latest version of fs-repo-migrations: %s", err) + } + + dir, err := ioutil.TempDir("", "go-ipfs-migrate") + if err != nil { + return "", fmt.Errorf("tempdir: %s", err) + } + + out := filepath.Join(dir, migrations) + + err = GetBinaryForVersion(migrations, migrations, DistPath, latest, out) + if err != nil { + fmt.Printf(" => error getting migrations binary: %s\n", err) + fmt.Println(" => could not find or install fs-repo-migrations, please manually install it") + return "", fmt.Errorf("failed to find migrations binary") + } + + err = os.Chmod(out, 0755) + if err != nil { + return "", err + } + + return out, nil +} + +func verifyMigrationSupportsVersion(fsrbin string, vn int) error { + sn, err := migrationsVersion(fsrbin) + if err != nil { + return err + } + + if sn >= vn { + return nil + } + + return fmt.Errorf("migrations binary doesnt support version %d", vn) +} + +func migrationsVersion(bin string) (int, error) { + out, err := exec.Command(bin, "-v").CombinedOutput() + if err != nil { + return 0, fmt.Errorf("failed to check migrations version: %s", err) + } + + vs := strings.Trim(string(out), " \n\t") + vn, err := strconv.Atoi(vs) + if err != nil { + return 0, fmt.Errorf("migrations binary version check did not return a number") + } + + return vn, nil +} + +func GetVersions(ipfspath, dist string) ([]string, error) { + rc, err := httpFetch(ipfspath + "/" + dist + "/versions") + if err != nil { + return nil, err + } + defer rc.Close() + + var out []string + scan := bufio.NewScanner(rc) + for scan.Scan() { + out = append(out, scan.Text()) + } + + return out, nil +} + +func GetLatestVersion(ipfspath, dist string) (string, error) { + vs, err := GetVersions(ipfspath, dist) + if err != nil { + return "", err + } + var latest string + for i := len(vs) - 1; i >= 0; i-- { + if !strings.Contains(vs[i], "-dev") { + latest = vs[i] + break + } + } + if latest == "" { + return "", fmt.Errorf("couldnt find a non dev version in the list") + } + return vs[len(vs)-1], nil +} + +func httpGet(url string) (*http.Response, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, fmt.Errorf("http.NewRequest error: %s", err) + } + + req.Header.Set("User-Agent", "go-ipfs") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("http.DefaultClient.Do error: %s", err) + } + + return resp, nil +} + +func httpFetch(url string) (io.ReadCloser, error) { + fmt.Printf("fetching url: %s\n", url) + resp, err := httpGet(url) + if err != nil { + return nil, err + } + + if resp.StatusCode >= 400 { + mes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error reading error body: %s", err) + } + + return nil, fmt.Errorf("%s: %s", resp.Status, string(mes)) + } + + return resp.Body, nil +} + +func GetBinaryForVersion(distname, binnom, root, vers, out string) error { + dir, err := ioutil.TempDir("", "go-ipfs-auto-migrate") + if err != nil { + return err + } + + var archive string + switch runtime.GOOS { + case "windows": + archive = "zip" + default: + archive = "tar.gz" + } + finame := fmt.Sprintf("%s_%s_%s-%s.%s", distname, vers, runtime.GOOS, runtime.GOARCH, archive) + distpath := fmt.Sprintf("%s/%s/%s/%s", root, distname, vers, finame) + + data, err := httpFetch(distpath) + if err != nil { + return err + } + + arcpath := filepath.Join(dir, finame) + fi, err := os.Create(arcpath) + if err != nil { + return err + } + + _, err = io.Copy(fi, data) + if err != nil { + return err + } + fi.Close() + + return unpackArchive(distname, binnom, arcpath, out, archive) +} diff --git a/repo/fsrepo/migrations/unpack.go b/repo/fsrepo/migrations/unpack.go new file mode 100644 index 00000000000..739564044a2 --- /dev/null +++ b/repo/fsrepo/migrations/unpack.go @@ -0,0 +1,101 @@ +package mfsr + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "fmt" + "io" + "os" +) + +func unpackArchive(dist, binnom, path, out, atype string) error { + switch atype { + case "zip": + return unpackZip(dist, binnom, path, out) + case "tar.gz": + return unpackTgz(dist, binnom, path, out) + default: + return fmt.Errorf("unrecognized archive type: %s", atype) + } +} + +func unpackTgz(dist, binnom, path, out string) error { + fi, err := os.Open(path) + if err != nil { + return err + } + defer fi.Close() + + gzr, err := gzip.NewReader(fi) + if err != nil { + return err + } + + defer gzr.Close() + + var bin io.Reader + tarr := tar.NewReader(gzr) + +loop: + for { + th, err := tarr.Next() + switch err { + default: + return err + case io.EOF: + break loop + case nil: + // continue + } + + if th.Name == dist+"/"+binnom { + bin = tarr + break + } + } + + if bin == nil { + return fmt.Errorf("no binary found in downloaded archive") + } + + return writeToPath(bin, out) +} + +func writeToPath(rc io.Reader, out string) error { + binfi, err := os.Create(out) + if err != nil { + return fmt.Errorf("error opening tmp bin path '%s': %s", out, err) + } + defer binfi.Close() + + _, err = io.Copy(binfi, rc) + if err != nil { + return err + } + + return nil +} + +func unpackZip(dist, binnom, path, out string) error { + zipr, err := zip.OpenReader(path) + if err != nil { + return fmt.Errorf("error opening zipreader: %s", err) + } + + defer zipr.Close() + + var bin io.ReadCloser + for _, fis := range zipr.File { + if fis.Name == dist+"/"+binnom+".exe" { + rc, err := fis.Open() + if err != nil { + return fmt.Errorf("error extracting binary from archive: %s", err) + } + + bin = rc + } + } + + return writeToPath(bin, out) +} From 2db7522fe82160be67f9e32226cb7082f60867f6 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 5 Jul 2016 23:26:03 -0700 Subject: [PATCH 2/7] make dist path overrideable License: MIT Signed-off-by: Jeromy --- repo/fsrepo/migrations/migrations.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/repo/fsrepo/migrations/migrations.go b/repo/fsrepo/migrations/migrations.go index b6558d510de..f2125531645 100644 --- a/repo/fsrepo/migrations/migrations.go +++ b/repo/fsrepo/migrations/migrations.go @@ -16,6 +16,12 @@ import ( var DistPath = "https://ipfs.io/ipns/dist.ipfs.io" +func init() { + if dist := os.Getenv("IPFS_DIST_PATH"); dist != "" { + DistPath = dist + } +} + const migrations = "fs-repo-migrations" func RunMigration(newv int) error { From cc73137f741212059b924bca406e3746c8e2565a Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 6 Jul 2016 10:29:31 -0700 Subject: [PATCH 3/7] add migrate flag to daemon subcommand License: MIT Signed-off-by: Jeromy --- cmd/ipfs/daemon.go | 42 +++++++++++++++++++++++++++++++++--------- repo/fsrepo/fsrepo.go | 18 ++++-------------- repo/mock.go | 2 +- 3 files changed, 38 insertions(+), 24 deletions(-) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 976550036f5..5ceb5b79f7e 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -23,6 +23,7 @@ import ( "github.com/ipfs/go-ipfs/core/corerouting" nodeMount "github.com/ipfs/go-ipfs/fuse/node" fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" + migrate "github.com/ipfs/go-ipfs/repo/fsrepo/migrations" pstore "gx/ipfs/QmQdnfvZQuhdT93LNc5bos52wAmdr3G2p6G8teLJMEN32P/go-libp2p-peerstore" conn "gx/ipfs/QmVCe3SNMjkcPgnpFhZs719dheq6xE7gJwjzV7aWcUM4Ms/go-libp2p/p2p/net/conn" util "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util" @@ -30,18 +31,19 @@ import ( ) const ( + adjustFDLimitKwd = "manage-fdlimit" + enableGCKwd = "enable-gc" initOptionKwd = "init" - routingOptionKwd = "routing" - routingOptionSupernodeKwd = "supernode" - mountKwd = "mount" - writableKwd = "writable" ipfsMountKwd = "mount-ipfs" ipnsMountKwd = "mount-ipns" - unrestrictedApiAccessKwd = "unrestricted-api" - unencryptTransportKwd = "disable-transport-encryption" - enableGCKwd = "enable-gc" - adjustFDLimitKwd = "manage-fdlimit" + migrateKwd = "migrate" + mountKwd = "mount" offlineKwd = "offline" + routingOptionKwd = "routing" + routingOptionSupernodeKwd = "supernode" + unencryptTransportKwd = "disable-transport-encryption" + unrestrictedApiAccessKwd = "unrestricted-api" + writableKwd = "writable" // apiAddrKwd = "address-api" // swarmAddrKwd = "address-swarm" ) @@ -139,6 +141,7 @@ Headers. cmds.BoolOption(enableGCKwd, "Enable automatic periodic repo garbage collection").Default(false), cmds.BoolOption(adjustFDLimitKwd, "Check and raise file descriptor limits if needed").Default(true), cmds.BoolOption(offlineKwd, "Run offline. Do not connect to the rest of the network but provide local API.").Default(false), + cmds.BoolOption(migrateKwd, "If true, assume yes at the migrate prompt. If false, assume no."), // TODO: add way to override addresses. tricky part: updating the config if also --init. // cmds.StringOption(apiAddrKwd, "Address for the daemon rpc API (overrides config)"), @@ -216,9 +219,30 @@ func daemonFunc(req cmds.Request, res cmds.Response) { // acquire the repo lock _before_ constructing a node. we need to make // sure we are permitted to access the resources (datastore, etc.) repo, err := fsrepo.Open(req.InvocContext().ConfigRoot) - if err != nil { + switch err { + default: res.SetError(err, cmds.ErrNormal) return + case fsrepo.ErrNeedMigration: + domigrate, found, _ := req.Option(migrateKwd).Bool() + + if !found { + err = migrate.TryMigrating(fsrepo.RepoVersion) + } else if domigrate { + err = migrate.RunMigration(fsrepo.RepoVersion) + } + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + repo, err = fsrepo.Open(req.InvocContext().ConfigRoot) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + case nil: + break } cfg, err := ctx.GetConfig() diff --git a/repo/fsrepo/fsrepo.go b/repo/fsrepo/fsrepo.go index e6ae0d384d0..45963ad5198 100644 --- a/repo/fsrepo/fsrepo.go +++ b/repo/fsrepo/fsrepo.go @@ -43,8 +43,9 @@ a migration in reverse. See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md for details.` var ( - ErrNoVersion = errors.New("no version file found, please run 0-to-1 migration tool.\n" + migrationInstructions) - ErrOldRepo = errors.New("ipfs repo found in old '~/.go-ipfs' location, please run migration tool.\n" + migrationInstructions) + ErrNoVersion = errors.New("no version file found, please run 0-to-1 migration tool.\n" + migrationInstructions) + ErrOldRepo = errors.New("ipfs repo found in old '~/.go-ipfs' location, please run migration tool.\n" + migrationInstructions) + ErrNeedMigration = errors.New("ipfs repo needs migration.") ) type NoRepoError struct { @@ -141,18 +142,7 @@ func open(repoPath string) (repo.Repo, error) { } if RepoVersion > ver { - r.lockfile.Close() - - err := mfsr.TryMigrating(RepoVersion) - if err != nil { - return nil, err - } - - r.lockfile, err = lockfile.Lock(r.path) - if err != nil { - return nil, fmt.Errorf("reacquiring lock: %s", err) - } - + return nil, ErrNeedMigration } else if ver > RepoVersion { // program version too low for existing repo return nil, fmt.Errorf(programTooLowMessage, RepoVersion, ver) diff --git a/repo/mock.go b/repo/mock.go index bd8e72af87d..8190a0bda1b 100644 --- a/repo/mock.go +++ b/repo/mock.go @@ -6,7 +6,7 @@ import ( "github.com/ipfs/go-ipfs/repo/config" ) -var errTODO = errors.New("TODO") +var errTODO = errors.New("TODO: mock repo") // Mock is not thread-safe type Mock struct { From 37673c85c700d5e4c9b9e1e89c7dff38c9b95c14 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 6 Jul 2016 15:22:59 -0700 Subject: [PATCH 4/7] add a test for auto migrations cli interface License: MIT Signed-off-by: Jeromy --- cmd/ipfs/daemon.go | 1 + repo/fsrepo/migrations/mfsr.go | 8 +++-- repo/fsrepo/migrations/migrations.go | 4 ++- test/sharness/t0066-migration.sh | 52 ++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 3 deletions(-) create mode 100755 test/sharness/t0066-migration.sh diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 5ceb5b79f7e..19013715062 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -225,6 +225,7 @@ func daemonFunc(req cmds.Request, res cmds.Response) { return case fsrepo.ErrNeedMigration: domigrate, found, _ := req.Option(migrateKwd).Bool() + fmt.Println("Found old repo version, migrations need to be run.") if !found { err = migrate.TryMigrating(fsrepo.RepoVersion) diff --git a/repo/fsrepo/migrations/mfsr.go b/repo/fsrepo/migrations/mfsr.go index 1370c183281..9f267927505 100644 --- a/repo/fsrepo/migrations/mfsr.go +++ b/repo/fsrepo/migrations/mfsr.go @@ -61,7 +61,7 @@ func (v VersionFileNotFound) Error() string { } func TryMigrating(tovers int) error { - if !YesNoPrompt("run migrations automatically? [y/n]") { + if !YesNoPrompt("Run migrations automatically? [y/N]") { return fmt.Errorf("please run the migrations manually") } @@ -70,7 +70,7 @@ func TryMigrating(tovers int) error { func YesNoPrompt(prompt string) bool { var s string - for { + for i := 0; i < 3; i++ { fmt.Printf("%s ", prompt) fmt.Scanf("%s", &s) switch s { @@ -78,7 +78,11 @@ func YesNoPrompt(prompt string) bool { return true case "n", "N": return false + case "": + return false } fmt.Println("Please press either 'y' or 'n'") } + + return false } diff --git a/repo/fsrepo/migrations/migrations.go b/repo/fsrepo/migrations/migrations.go index f2125531645..f33e9a3b9fe 100644 --- a/repo/fsrepo/migrations/migrations.go +++ b/repo/fsrepo/migrations/migrations.go @@ -27,7 +27,9 @@ const migrations = "fs-repo-migrations" func RunMigration(newv int) error { migrateBin := "fs-repo-migrations" fmt.Println(" => checking for migrations binary...") - _, err := exec.LookPath(migrateBin) + + var err error + migrateBin, err = exec.LookPath(migrateBin) if err == nil { // check to make sure migrations binary supports our target version err = verifyMigrationSupportsVersion(migrateBin, newv) diff --git a/test/sharness/t0066-migration.sh b/test/sharness/t0066-migration.sh new file mode 100755 index 00000000000..8ff002b78f1 --- /dev/null +++ b/test/sharness/t0066-migration.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# +# Copyright (c) 2016 Jeromy Johnson +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test migrations auto update prompt" + +. lib/test-lib.sh + +test_init_ipfs + +test_expect_success "setup mock migrations" ' + mkdir bin && + echo "#!/bin/bash" > bin/fs-repo-migrations && + echo "echo 4" >> bin/fs-repo-migrations && + chmod +x bin/fs-repo-migrations && + export PATH="$(pwd)/bin":$PATH +' + +test_expect_success "manually reset repo version to 3" ' + echo "3" > "$IPFS_PATH"/version +' + +test_expect_success "ipfs daemon --migrate=false fails" ' + test_expect_code 1 ipfs daemon --migrate=false 2> false_out +' + +test_expect_success "output looks good" ' + grep "ipfs repo needs migration" false_out +' + +test_expect_success "ipfs daemon --migrate=true runs migration" ' + test_expect_code 1 ipfs daemon --migrate=true > true_out +' + +test_expect_success "output looks good" ' + grep "running migration" true_out > /dev/null && + grep "binary completed successfully" true_out > /dev/null +' + +test_expect_success "'ipfs daemon' prompts to auto migrate" ' + test_expect_code 1 ipfs daemon > daemon_out 2> daemon_err +' + +test_expect_success "output looks good" ' + grep "Found old repo version" daemon_out > /dev/null && + grep "Run migrations automatically?" daemon_out > /dev/null && + grep "please run the migrations manually" daemon_err > /dev/null +' + +test_done From 8a8730fa4032942ec5181341104d2e75135af05c Mon Sep 17 00:00:00 2001 From: Lars Gierth Date: Sun, 10 Jul 2016 03:53:55 +0200 Subject: [PATCH 5/7] repo: detect OS variant, for now only linux-musl License: MIT Signed-off-by: Lars Gierth --- repo/fsrepo/migrations/migrations.go | 39 +++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/repo/fsrepo/migrations/migrations.go b/repo/fsrepo/migrations/migrations.go index f33e9a3b9fe..8709bc11186 100644 --- a/repo/fsrepo/migrations/migrations.go +++ b/repo/fsrepo/migrations/migrations.go @@ -2,6 +2,7 @@ package mfsr import ( "bufio" + "bytes" "fmt" "io" "io/ioutil" @@ -44,7 +45,7 @@ func RunMigration(newv int) error { err = verifyMigrationSupportsVersion(loc, newv) if err != nil { - return fmt.Errorf("could not find migrations binary that supports version %d", newv) + return fmt.Errorf("no migration binary found that supports version %d - %s", newv, err) } migrateBin = loc @@ -104,7 +105,7 @@ func verifyMigrationSupportsVersion(fsrbin string, vn int) error { return nil } - return fmt.Errorf("migrations binary doesnt support version %d", vn) + return fmt.Errorf("migrations binary doesnt support version %d: %s", vn, fsrbin) } func migrationsVersion(bin string) (int, error) { @@ -204,7 +205,11 @@ func GetBinaryForVersion(distname, binnom, root, vers, out string) error { default: archive = "tar.gz" } - finame := fmt.Sprintf("%s_%s_%s-%s.%s", distname, vers, runtime.GOOS, runtime.GOARCH, archive) + osv, err := osWithVariant() + if err != nil { + return err + } + finame := fmt.Sprintf("%s_%s_%s-%s.%s", distname, vers, osv, runtime.GOARCH, archive) distpath := fmt.Sprintf("%s/%s/%s/%s", root, distname, vers, finame) data, err := httpFetch(distpath) @@ -226,3 +231,31 @@ func GetBinaryForVersion(distname, binnom, root, vers, out string) error { return unpackArchive(distname, binnom, arcpath, out, archive) } + +func osWithVariant() (string, error) { + if runtime.GOOS != "linux" { + return runtime.GOOS, nil + } + + bin, err := exec.LookPath(filepath.Base(os.Args[0])) + if err != nil { + return "", fmt.Errorf("failed to resolve go-ipfs: %s", err) + } + + cmd := exec.Command("ldd", bin) + buf := new(bytes.Buffer) + cmd.Stdout = buf + err = cmd.Run() + if err != nil { + return "", fmt.Errorf("failed to run ldd: %s", err) + } + + scan := bufio.NewScanner(buf) + for scan.Scan() { + if strings.Contains(scan.Text(), "libc") && strings.Contains(scan.Text(), "musl") { + return "linux-musl", nil + } + } + + return "linux", nil +} From fd732d2db4e95d8e1cb7b401bba47112af404435 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 19 Jul 2016 06:49:05 -0700 Subject: [PATCH 6/7] hardcode in dists page path License: MIT Signed-off-by: Jeromy --- repo/fsrepo/migrations/migrations.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo/fsrepo/migrations/migrations.go b/repo/fsrepo/migrations/migrations.go index 8709bc11186..b3976e6d133 100644 --- a/repo/fsrepo/migrations/migrations.go +++ b/repo/fsrepo/migrations/migrations.go @@ -15,7 +15,7 @@ import ( "strings" ) -var DistPath = "https://ipfs.io/ipns/dist.ipfs.io" +var DistPath = "https://ipfs.io/ipfs/QmUnvqDuRyfe7HJuiMMHv77AMUFnjGyAU28LFPeTYwGmFF" func init() { if dist := os.Getenv("IPFS_DIST_PATH"); dist != "" { From fcc4f0001d968ced8db8fcce12327a89a97e7d1b Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 20 Jul 2016 03:14:29 -0700 Subject: [PATCH 7/7] move prompt code into daemon.go License: MIT Signed-off-by: Jeromy --- cmd/ipfs/daemon.go | 30 +++++++++++++++++++++++++++--- repo/fsrepo/migrations/mfsr.go | 27 --------------------------- test/sharness/t0066-migration.sh | 2 +- 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 19013715062..24e42093caa 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -228,10 +228,15 @@ func daemonFunc(req cmds.Request, res cmds.Response) { fmt.Println("Found old repo version, migrations need to be run.") if !found { - err = migrate.TryMigrating(fsrepo.RepoVersion) - } else if domigrate { - err = migrate.RunMigration(fsrepo.RepoVersion) + domigrate = YesNoPrompt("Run migrations automatically? [y/N]") } + + if !domigrate { + res.SetError(fmt.Errorf("please run the migrations manually"), cmds.ErrNormal) + return + } + + err = migrate.RunMigration(fsrepo.RepoVersion) if err != nil { res.SetError(err, cmds.ErrNormal) return @@ -594,3 +599,22 @@ func merge(cs ...<-chan error) <-chan error { }() return out } + +func YesNoPrompt(prompt string) bool { + var s string + for i := 0; i < 3; i++ { + fmt.Printf("%s ", prompt) + fmt.Scanf("%s", &s) + switch s { + case "y", "Y": + return true + case "n", "N": + return false + case "": + return false + } + fmt.Println("Please press either 'y' or 'n'") + } + + return false +} diff --git a/repo/fsrepo/migrations/mfsr.go b/repo/fsrepo/migrations/mfsr.go index 9f267927505..3e9329f17ca 100644 --- a/repo/fsrepo/migrations/mfsr.go +++ b/repo/fsrepo/migrations/mfsr.go @@ -59,30 +59,3 @@ type VersionFileNotFound string func (v VersionFileNotFound) Error() string { return "no version file in repo at " + string(v) } - -func TryMigrating(tovers int) error { - if !YesNoPrompt("Run migrations automatically? [y/N]") { - return fmt.Errorf("please run the migrations manually") - } - - return RunMigration(tovers) -} - -func YesNoPrompt(prompt string) bool { - var s string - for i := 0; i < 3; i++ { - fmt.Printf("%s ", prompt) - fmt.Scanf("%s", &s) - switch s { - case "y", "Y": - return true - case "n", "N": - return false - case "": - return false - } - fmt.Println("Please press either 'y' or 'n'") - } - - return false -} diff --git a/test/sharness/t0066-migration.sh b/test/sharness/t0066-migration.sh index 8ff002b78f1..78689f32367 100755 --- a/test/sharness/t0066-migration.sh +++ b/test/sharness/t0066-migration.sh @@ -27,7 +27,7 @@ test_expect_success "ipfs daemon --migrate=false fails" ' ' test_expect_success "output looks good" ' - grep "ipfs repo needs migration" false_out + grep "please run the migrations manually" false_out ' test_expect_success "ipfs daemon --migrate=true runs migration" '