diff --git a/fs-repo-11-to-12/go.mod b/fs-repo-11-to-12/go.mod index 6a666f19..0d2769d8 100644 --- a/fs-repo-11-to-12/go.mod +++ b/fs-repo-11-to-12/go.mod @@ -7,7 +7,7 @@ require ( github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea github.com/ipfs/go-cid v0.0.7 github.com/ipfs/go-datastore v0.4.5 - github.com/ipfs/go-ds-badger v0.2.7-0.20211210151007-a2805355dcf5 // indirect + github.com/ipfs/go-ds-badger v0.2.7-0.20220117180822-159330558612 // indirect github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-ipfs v0.8.0 github.com/ipfs/go-ipfs-ds-help v1.0.0 diff --git a/fs-repo-11-to-12/go.sum b/fs-repo-11-to-12/go.sum index a71c85a3..d68c6327 100644 --- a/fs-repo-11-to-12/go.sum +++ b/fs-repo-11-to-12/go.sum @@ -295,8 +295,8 @@ github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9 github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE= github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk= github.com/ipfs/go-ds-badger v0.2.6/go.mod h1:02rnztVKA4aZwDuaRPTf8mpqcKmXP7mLl6JPxd14JHA= -github.com/ipfs/go-ds-badger v0.2.7-0.20211210151007-a2805355dcf5 h1:ovdpQk2ZVK6eQLzMCZy1z2tJae7yvE8xaPUw4Pr1RqI= -github.com/ipfs/go-ds-badger v0.2.7-0.20211210151007-a2805355dcf5/go.mod h1:02rnztVKA4aZwDuaRPTf8mpqcKmXP7mLl6JPxd14JHA= +github.com/ipfs/go-ds-badger v0.2.7-0.20220117180822-159330558612 h1:Uvp2/ZNlR3YmH04XJGv4YsPabhPzRPyeroLltVKBSr8= +github.com/ipfs/go-ds-badger v0.2.7-0.20220117180822-159330558612/go.mod h1:02rnztVKA4aZwDuaRPTf8mpqcKmXP7mLl6JPxd14JHA= github.com/ipfs/go-ds-flatfs v0.4.5 h1:4QceuKEbH+HVZ2ZommstJMi3o3II+dWS3IhLaD7IGHs= github.com/ipfs/go-ds-flatfs v0.4.5/go.mod h1:e4TesLyZoA8k1gV/yCuBTnt2PJtypn4XUlB5n8KQMZY= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= diff --git a/fs-repo-11-to-12/migration/swapper.go b/fs-repo-11-to-12/migration/swapper.go index df029ec9..aa6e6909 100644 --- a/fs-repo-11-to-12/migration/swapper.go +++ b/fs-repo-11-to-12/migration/swapper.go @@ -2,6 +2,8 @@ package mg11 import ( "errors" + "os" + "strconv" "sync" "sync/atomic" @@ -19,6 +21,32 @@ var SyncSize uint64 = 20 * 1024 * 1024 // 20MiB // migration. var NWorkers int = 4 +func init() { + workerEnvVar := "IPFS_FS_MIGRATION_11_TO_12_NWORKERS" + syncSizeEnvVar := "IPFS_FS_MIGRATION_11_TO_12_SYNC_SIZE_BYTES" + if nworkersStr, nworkerInEnv := os.LookupEnv(workerEnvVar); nworkerInEnv { + nworkers, err := strconv.Atoi(nworkersStr) + if err != nil { + panic(err) + } + if nworkers < 1 { + panic("number of workers must be at least 1") + } + NWorkers = nworkers + } + + if syncSizeStr, syncSizeInEnv := os.LookupEnv(syncSizeEnvVar); syncSizeInEnv { + syncSize, err := strconv.ParseUint(syncSizeStr, 10, 64) + if err != nil { + panic(err) + } + if syncSize < 1 { + panic("sync size bytes must be at least 1") + } + SyncSize = syncSize + } +} + // Swap holds the datastore keys for the original CID and for the // destination Multihash. type Swap struct { @@ -114,9 +142,11 @@ func (cswap *CidSwapper) swapWorker(dryRun bool, resultsCh <-chan query.Result) oldKey := ds.NewKey(res.Key) c, err := dsKeyToCid(ds.NewKey(oldKey.BaseNamespace())) // remove prefix if err != nil { - // complain if we find anything that is not a CID but + // complain if we find anything that is not a CID or multihash but // leave it as it is. - log.Log("could not parse %s as a Cid", oldKey) + if _, err := dshelp.DsKeyToMultihash(ds.NewKey(oldKey.BaseNamespace())); err != nil { + log.Log("could not parse %s as a Cid or Multihash", oldKey) + } continue } if c.Version() == 0 { // CidV0 are multihashes, leave them. diff --git a/fs-repo-11-to-12/vendor/github.com/ipfs/go-ds-badger/datastore.go b/fs-repo-11-to-12/vendor/github.com/ipfs/go-ds-badger/datastore.go index 34780b4f..6d68dc16 100644 --- a/fs-repo-11-to-12/vendor/github.com/ipfs/go-ds-badger/datastore.go +++ b/fs-repo-11-to-12/vendor/github.com/ipfs/go-ds-badger/datastore.go @@ -87,7 +87,7 @@ var DefaultOptions Options func init() { DefaultOptions = Options{ - GcDiscardRatio: 0.2, + GcDiscardRatio: 0.5, GcInterval: 2 * time.Minute, GcSleep: 10 * time.Second, Options: badger.LSMOnlyOptions(""), diff --git a/fs-repo-11-to-12/vendor/modules.txt b/fs-repo-11-to-12/vendor/modules.txt index d4b3b274..157a1c19 100644 --- a/fs-repo-11-to-12/vendor/modules.txt +++ b/fs-repo-11-to-12/vendor/modules.txt @@ -123,7 +123,7 @@ github.com/ipfs/go-datastore/mount github.com/ipfs/go-datastore/namespace github.com/ipfs/go-datastore/query github.com/ipfs/go-datastore/sync -# github.com/ipfs/go-ds-badger v0.2.7-0.20211210151007-a2805355dcf5 +# github.com/ipfs/go-ds-badger v0.2.7-0.20220117180822-159330558612 ## explicit github.com/ipfs/go-ds-badger # github.com/ipfs/go-ds-flatfs v0.4.5 diff --git a/sharness/t0030-simple-migration.sh b/sharness/t0030-simple-migration.sh index 9b368210..35127200 100755 --- a/sharness/t0030-simple-migration.sh +++ b/sharness/t0030-simple-migration.sh @@ -4,6 +4,8 @@ test_description="Simple fs-repo-migrations tests" . lib/test-lib.sh +latestRepoVersion="12" + test_expect_success "fs-repo-migrations binary is here" ' test -f "$LOCAL_FS_REPO_MIG" ' @@ -13,7 +15,7 @@ test_expect_success "'fs-repo-migrations -v' works" ' ' test_expect_success "'fs-repo-migrations -v' output looks good" ' - echo "11" >expected && + echo "$latestRepoVersion" >expected && test_cmp expected actual ' @@ -30,7 +32,7 @@ test_expect_success "'fs-repo-migrations -v' works" ' ' test_expect_success "'fs-repo-migrations -v' output looks good" ' - echo "11" >expected && + echo "$latestRepoVersion" >expected && test_cmp expected actual '