From 0b1276db596e9d1cfdf1bc584f8ee3344a76910d Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 4 Dec 2020 15:28:54 -0500 Subject: [PATCH 1/3] Use datastore based pinner --- core/node/core.go | 16 +++++++--------- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/core/node/core.go b/core/node/core.go index 4c522dbcfc3..cb757d81968 100644 --- a/core/node/core.go +++ b/core/node/core.go @@ -3,6 +3,7 @@ package node import ( "context" "fmt" + "time" "github.com/ipfs/go-bitswap" "github.com/ipfs/go-bitswap/network" @@ -12,8 +13,8 @@ import ( "github.com/ipfs/go-filestore" "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-ipfs-exchange-interface" - "github.com/ipfs/go-ipfs-exchange-offline" "github.com/ipfs/go-ipfs-pinner" + "github.com/ipfs/go-ipfs-pinner/dspinner" "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" "github.com/ipfs/go-mfs" @@ -41,7 +42,6 @@ func BlockService(lc fx.Lifecycle, bs blockstore.Blockstore, rem exchange.Interf // Pinning creates new pinner which tells GC which blocks should be kept func Pinning(bstore blockstore.Blockstore, ds format.DAGService, repo repo.Repo) (pin.Pinner, error) { - internalDag := merkledag.NewDAGService(blockservice.New(bstore, offline.Exchange(bstore))) rootDS := repo.Datastore() syncFn := func() error { @@ -51,15 +51,13 @@ func Pinning(bstore blockstore.Blockstore, ds format.DAGService, repo repo.Repo) return rootDS.Sync(filestore.FilestorePrefix) } syncDs := &syncDagService{ds, syncFn} - syncInternalDag := &syncDagService{internalDag, syncFn} - pinning, err := pin.LoadPinner(rootDS, syncDs, syncInternalDag) + ctx, cancel := context.WithTimeout(context.TODO(), 2*time.Minute) + defer cancel() + + pinning, err := dspinner.New(ctx, rootDS, syncDs) if err != nil { - // TODO: we should move towards only running 'NewPinner' explicitly on - // node init instead of implicitly here as a result of the pinner keys - // not being found in the datastore. - // this is kinda sketchy and could cause data loss - pinning = pin.NewPinner(rootDS, syncDs, syncInternalDag) + return nil, err } return pinning, nil diff --git a/go.mod b/go.mod index 5d5bb900f76..8c43c8c87d4 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/ipfs/go-ipfs-exchange-interface v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 github.com/ipfs/go-ipfs-files v0.0.8 - github.com/ipfs/go-ipfs-pinner v0.0.4 + github.com/ipfs/go-ipfs-pinner v0.1.0 github.com/ipfs/go-ipfs-posinfo v0.0.1 github.com/ipfs/go-ipfs-provider v0.4.3 github.com/ipfs/go-ipfs-routing v0.1.0 diff --git a/go.sum b/go.sum index aefb59800d3..f49ce66adf9 100644 --- a/go.sum +++ b/go.sum @@ -388,8 +388,8 @@ github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjN github.com/ipfs/go-ipfs-files v0.0.8 h1:8o0oFJkJ8UkO/ABl8T6ac6tKF3+NIpj67aAB6ZpusRg= github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs= github.com/ipfs/go-ipfs-flags v0.0.1/go.mod h1:RnXBb9WV53GSfTrSDVK61NLTFKvWc60n+K9EgCDh+rA= -github.com/ipfs/go-ipfs-pinner v0.0.4 h1:EmxhS3vDsCK/rZrsgxX0Le9m2drBcGlUd7ah/VyFYVE= -github.com/ipfs/go-ipfs-pinner v0.0.4/go.mod h1:s4kFZWLWGDudN8Jyd/GTpt222A12C2snA2+OTdy/7p8= +github.com/ipfs/go-ipfs-pinner v0.1.0 h1:rjSrbUDYd1YYHZ5dOgu+QEOuLcU0m/2a/brcxC/ReeU= +github.com/ipfs/go-ipfs-pinner v0.1.0/go.mod h1:EzyyaWCWeZJ/he9cDBH6QrEkSuRqTRWMmCoyNkylTTg= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A= github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= From 9bc8f6d5b514cb599cbb1999ca22c933f7d77d50 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 4 Dec 2020 15:30:04 -0500 Subject: [PATCH 2/3] bump repo version to 11 --- repo/fsrepo/fsrepo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo/fsrepo/fsrepo.go b/repo/fsrepo/fsrepo.go index 6cd312e40aa..011c47a1d7c 100644 --- a/repo/fsrepo/fsrepo.go +++ b/repo/fsrepo/fsrepo.go @@ -35,7 +35,7 @@ const LockFile = "repo.lock" var log = logging.Logger("fsrepo") // version number that we are currently expecting to see -var RepoVersion = 10 +var RepoVersion = 11 var migrationInstructions = `See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md Sorry for the inconvenience. In the future, these will run automatically.` From 8a05c59d3dc6205e72c9c59b40cda0d10efa4a64 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 8 Dec 2020 14:46:00 -0500 Subject: [PATCH 3/3] chore: update dist path --- repo/fsrepo/migrations/migrations.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo/fsrepo/migrations/migrations.go b/repo/fsrepo/migrations/migrations.go index 186d4a10dc0..ad94b04a652 100644 --- a/repo/fsrepo/migrations/migrations.go +++ b/repo/fsrepo/migrations/migrations.go @@ -15,7 +15,7 @@ import ( "strings" ) -var DistPath = "https://ipfs.io/ipfs/Qmdo5m6bpQXCayzfGghyvgXJdVHSsXsCKDUo9vWktDKq3K" +var DistPath = "https://ipfs.io/ipfs/QmahS3i7MkjVVDjPs6CoPhCKGi27LkHGXrzQ45jCRX6wcK" func init() { if dist := os.Getenv("IPFS_DIST_PATH"); dist != "" {