Merge pull request #7750 from ipfs/feat/datastore-pinner

Use datastore based pinning

core/node/core.go

@@ -3,6 +3,7 @@ package node
 import (
  "context"
  "fmt"
+ "time"
 
  "github.com/ipfs/go-bitswap"
  "github.com/ipfs/go-bitswap/network"
@@ -12,8 +13,8 @@ import (
  "github.com/ipfs/go-filestore"
  "github.com/ipfs/go-ipfs-blockstore"
  "github.com/ipfs/go-ipfs-exchange-interface"
- "github.com/ipfs/go-ipfs-exchange-offline"
  "github.com/ipfs/go-ipfs-pinner"
+ "github.com/ipfs/go-ipfs-pinner/dspinner"
  "github.com/ipfs/go-ipld-format"
  "github.com/ipfs/go-merkledag"
  "github.com/ipfs/go-mfs"
@@ -41,7 +42,6 @@ func BlockService(lc fx.Lifecycle, bs blockstore.Blockstore, rem exchange.Interf
 
 // Pinning creates new pinner which tells GC which blocks should be kept
 func Pinning(bstore blockstore.Blockstore, ds format.DAGService, repo repo.Repo) (pin.Pinner, error) {
- internalDag := merkledag.NewDAGService(blockservice.New(bstore, offline.Exchange(bstore)))
  rootDS := repo.Datastore()
 
  syncFn := func() error {
@@ -51,15 +51,13 @@ func Pinning(bstore blockstore.Blockstore, ds format.DAGService, repo repo.Repo)
  return rootDS.Sync(filestore.FilestorePrefix)
  }
  syncDs := &syncDagService{ds, syncFn}
- syncInternalDag := &syncDagService{internalDag, syncFn}
 
- pinning, err := pin.LoadPinner(rootDS, syncDs, syncInternalDag)
+ ctx, cancel := context.WithTimeout(context.TODO(), 2*time.Minute)
+ defer cancel()
+
+ pinning, err := dspinner.New(ctx, rootDS, syncDs)
  if err != nil {
- // TODO: we should move towards only running 'NewPinner' explicitly on
- // node init instead of implicitly here as a result of the pinner keys
- // not being found in the datastore.
- // this is kinda sketchy and could cause data loss
- pinning = pin.NewPinner(rootDS, syncDs, syncInternalDag)
+ return nil, err
  }
 
  return pinning, nil

go.mod

@@ -36,7 +36,7 @@ require (
  github.com/ipfs/go-ipfs-exchange-interface v0.0.1
  github.com/ipfs/go-ipfs-exchange-offline v0.0.1
  github.com/ipfs/go-ipfs-files v0.0.8
- github.com/ipfs/go-ipfs-pinner v0.0.4
+ github.com/ipfs/go-ipfs-pinner v0.1.0
  github.com/ipfs/go-ipfs-posinfo v0.0.1
  github.com/ipfs/go-ipfs-provider v0.4.3
  github.com/ipfs/go-ipfs-routing v0.1.0

go.sum

@@ -388,8 +388,8 @@ github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjN
 github.com/ipfs/go-ipfs-files v0.0.8 h1:8o0oFJkJ8UkO/ABl8T6ac6tKF3+NIpj67aAB6ZpusRg=
 github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs=
 github.com/ipfs/go-ipfs-flags v0.0.1/go.mod h1:RnXBb9WV53GSfTrSDVK61NLTFKvWc60n+K9EgCDh+rA=
-github.com/ipfs/go-ipfs-pinner v0.0.4 h1:EmxhS3vDsCK/rZrsgxX0Le9m2drBcGlUd7ah/VyFYVE=
-github.com/ipfs/go-ipfs-pinner v0.0.4/go.mod h1:s4kFZWLWGDudN8Jyd/GTpt222A12C2snA2+OTdy/7p8=
+github.com/ipfs/go-ipfs-pinner v0.1.0 h1:rjSrbUDYd1YYHZ5dOgu+QEOuLcU0m/2a/brcxC/ReeU=
+github.com/ipfs/go-ipfs-pinner v0.1.0/go.mod h1:EzyyaWCWeZJ/he9cDBH6QrEkSuRqTRWMmCoyNkylTTg=
 github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs=
 github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A=
 github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY=

repo/fsrepo/fsrepo.go

@@ -35,7 +35,7 @@ const LockFile = "repo.lock"
 var log = logging.Logger("fsrepo")
 
 // version number that we are currently expecting to see
-var RepoVersion = 10
+var RepoVersion = 11
 
 var migrationInstructions = `See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md
 Sorry for the inconvenience. In the future, these will run automatically.`

repo/fsrepo/migrations/migrations.go

@@ -15,7 +15,7 @@ import (
  "strings"
 )
 
-var DistPath = "https://ipfs.io/ipfs/Qmdo5m6bpQXCayzfGghyvgXJdVHSsXsCKDUo9vWktDKq3K"
+var DistPath = "https://ipfs.io/ipfs/QmahS3i7MkjVVDjPs6CoPhCKGi27LkHGXrzQ45jCRX6wcK"
 
 func init() {
  if dist := os.Getenv("IPFS_DIST_PATH"); dist != "" {