From 4dd19441df0553c38a517450c2bb84857626c210 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 14 Jan 2022 13:40:36 -0500 Subject: [PATCH 1/4] fs-repo-11-to-12: add environment variables for configuring the number of workers and the sync size --- fs-repo-11-to-12/migration/swapper.go | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/fs-repo-11-to-12/migration/swapper.go b/fs-repo-11-to-12/migration/swapper.go index df029ec9..56d1ae78 100644 --- a/fs-repo-11-to-12/migration/swapper.go +++ b/fs-repo-11-to-12/migration/swapper.go @@ -2,6 +2,8 @@ package mg11 import ( "errors" + "os" + "strconv" "sync" "sync/atomic" @@ -19,6 +21,32 @@ var SyncSize uint64 = 20 * 1024 * 1024 // 20MiB // migration. var NWorkers int = 4 +func init() { + workerEnvVar := "IPFS_FS_MIGRATION_11_TO_12_NWORKERS" + syncSizeEnvVar := "IPFS_FS_MIGRATION_11_TO_12_SYNC_SIZE" + if nworkersStr, nworkerInEnv := os.LookupEnv(workerEnvVar); nworkerInEnv { + nworkers, err := strconv.Atoi(nworkersStr) + if err != nil { + panic(err) + } + if nworkers < 1 { + panic("number of workers must be at least 1") + } + NWorkers = nworkers + } + + if syncSizeStr, syncSizeInEnv := os.LookupEnv(syncSizeEnvVar); syncSizeInEnv { + syncSize, err := strconv.ParseUint(syncSizeStr, 10, 64) + if err != nil { + panic(err) + } + if syncSize < 1 { + panic("sync size bytes must be at least 1") + } + SyncSize = syncSize + } +} + // Swap holds the datastore keys for the original CID and for the // destination Multihash. type Swap struct { From 09055a5f34df3e76290984e5ef0c4f5796fc05ec Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 17 Jan 2022 13:11:20 -0500 Subject: [PATCH 2/4] use branch of go-ds-badger with increased GcDiscardRatio --- fs-repo-11-to-12/go.mod | 2 +- fs-repo-11-to-12/go.sum | 4 ++-- .../vendor/github.com/ipfs/go-ds-badger/datastore.go | 2 +- fs-repo-11-to-12/vendor/modules.txt | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fs-repo-11-to-12/go.mod b/fs-repo-11-to-12/go.mod index 6a666f19..0d2769d8 100644 --- a/fs-repo-11-to-12/go.mod +++ b/fs-repo-11-to-12/go.mod @@ -7,7 +7,7 @@ require ( github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea github.com/ipfs/go-cid v0.0.7 github.com/ipfs/go-datastore v0.4.5 - github.com/ipfs/go-ds-badger v0.2.7-0.20211210151007-a2805355dcf5 // indirect + github.com/ipfs/go-ds-badger v0.2.7-0.20220117180822-159330558612 // indirect github.com/ipfs/go-filestore v1.0.0 github.com/ipfs/go-ipfs v0.8.0 github.com/ipfs/go-ipfs-ds-help v1.0.0 diff --git a/fs-repo-11-to-12/go.sum b/fs-repo-11-to-12/go.sum index a71c85a3..d68c6327 100644 --- a/fs-repo-11-to-12/go.sum +++ b/fs-repo-11-to-12/go.sum @@ -295,8 +295,8 @@ github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9 github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE= github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk= github.com/ipfs/go-ds-badger v0.2.6/go.mod h1:02rnztVKA4aZwDuaRPTf8mpqcKmXP7mLl6JPxd14JHA= -github.com/ipfs/go-ds-badger v0.2.7-0.20211210151007-a2805355dcf5 h1:ovdpQk2ZVK6eQLzMCZy1z2tJae7yvE8xaPUw4Pr1RqI= -github.com/ipfs/go-ds-badger v0.2.7-0.20211210151007-a2805355dcf5/go.mod h1:02rnztVKA4aZwDuaRPTf8mpqcKmXP7mLl6JPxd14JHA= +github.com/ipfs/go-ds-badger v0.2.7-0.20220117180822-159330558612 h1:Uvp2/ZNlR3YmH04XJGv4YsPabhPzRPyeroLltVKBSr8= +github.com/ipfs/go-ds-badger v0.2.7-0.20220117180822-159330558612/go.mod h1:02rnztVKA4aZwDuaRPTf8mpqcKmXP7mLl6JPxd14JHA= github.com/ipfs/go-ds-flatfs v0.4.5 h1:4QceuKEbH+HVZ2ZommstJMi3o3II+dWS3IhLaD7IGHs= github.com/ipfs/go-ds-flatfs v0.4.5/go.mod h1:e4TesLyZoA8k1gV/yCuBTnt2PJtypn4XUlB5n8KQMZY= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= diff --git a/fs-repo-11-to-12/vendor/github.com/ipfs/go-ds-badger/datastore.go b/fs-repo-11-to-12/vendor/github.com/ipfs/go-ds-badger/datastore.go index 34780b4f..6d68dc16 100644 --- a/fs-repo-11-to-12/vendor/github.com/ipfs/go-ds-badger/datastore.go +++ b/fs-repo-11-to-12/vendor/github.com/ipfs/go-ds-badger/datastore.go @@ -87,7 +87,7 @@ var DefaultOptions Options func init() { DefaultOptions = Options{ - GcDiscardRatio: 0.2, + GcDiscardRatio: 0.5, GcInterval: 2 * time.Minute, GcSleep: 10 * time.Second, Options: badger.LSMOnlyOptions(""), diff --git a/fs-repo-11-to-12/vendor/modules.txt b/fs-repo-11-to-12/vendor/modules.txt index d4b3b274..157a1c19 100644 --- a/fs-repo-11-to-12/vendor/modules.txt +++ b/fs-repo-11-to-12/vendor/modules.txt @@ -123,7 +123,7 @@ github.com/ipfs/go-datastore/mount github.com/ipfs/go-datastore/namespace github.com/ipfs/go-datastore/query github.com/ipfs/go-datastore/sync -# github.com/ipfs/go-ds-badger v0.2.7-0.20211210151007-a2805355dcf5 +# github.com/ipfs/go-ds-badger v0.2.7-0.20220117180822-159330558612 ## explicit github.com/ipfs/go-ds-badger # github.com/ipfs/go-ds-flatfs v0.4.5 From 368fcabe6ea18a27b0e572e65c5e82f8c0968fcd Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 17 Jan 2022 16:09:48 -0500 Subject: [PATCH 3/4] test: update sharness to track latest released repo version --- sharness/t0030-simple-migration.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sharness/t0030-simple-migration.sh b/sharness/t0030-simple-migration.sh index 9b368210..35127200 100755 --- a/sharness/t0030-simple-migration.sh +++ b/sharness/t0030-simple-migration.sh @@ -4,6 +4,8 @@ test_description="Simple fs-repo-migrations tests" . lib/test-lib.sh +latestRepoVersion="12" + test_expect_success "fs-repo-migrations binary is here" ' test -f "$LOCAL_FS_REPO_MIG" ' @@ -13,7 +15,7 @@ test_expect_success "'fs-repo-migrations -v' works" ' ' test_expect_success "'fs-repo-migrations -v' output looks good" ' - echo "11" >expected && + echo "$latestRepoVersion" >expected && test_cmp expected actual ' @@ -30,7 +32,7 @@ test_expect_success "'fs-repo-migrations -v' works" ' ' test_expect_success "'fs-repo-migrations -v' output looks good" ' - echo "11" >expected && + echo "$latestRepoVersion" >expected && test_cmp expected actual ' From ff44fe3270a9ee77d1e51da884513ea14fa652a5 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 19 Jan 2022 09:33:41 +0100 Subject: [PATCH 4/4] Rename SYNC_SIZE env var to SYNC_SIZE_BYTES Avoid that there is confusion about the unit used. --- fs-repo-11-to-12/migration/swapper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs-repo-11-to-12/migration/swapper.go b/fs-repo-11-to-12/migration/swapper.go index 56d1ae78..38cca6b3 100644 --- a/fs-repo-11-to-12/migration/swapper.go +++ b/fs-repo-11-to-12/migration/swapper.go @@ -23,7 +23,7 @@ var NWorkers int = 4 func init() { workerEnvVar := "IPFS_FS_MIGRATION_11_TO_12_NWORKERS" - syncSizeEnvVar := "IPFS_FS_MIGRATION_11_TO_12_SYNC_SIZE" + syncSizeEnvVar := "IPFS_FS_MIGRATION_11_TO_12_SYNC_SIZE_BYTES" if nworkersStr, nworkerInEnv := os.LookupEnv(workerEnvVar); nworkerInEnv { nworkers, err := strconv.Atoi(nworkersStr) if err != nil {