piecedirectory: Cache piece readers in BlockstoreGet

.circleci/config.yml

@@ -1,6 +1,6 @@
 version: 2.1
 orbs:
-  go: gotest/tools@0.0.13
+  go: gotest/tools@0.0.14
 
 executors:
   golang:
@@ -83,6 +83,19 @@ jobs:
           paths:
             - linux
 
+  lid-docker-compose:
+    description: 'Run LID integration tests'
+    machine:
+      image: ubuntu-2004:202104-01
+      resource_class: xlarge
+    steps:
+      - checkout
+      - run:
+          name: local index directory docker compose tests
+          command: |
+            set -x
+            make test-lid
+
   test:
     description: |
       Run go tests
@@ -98,6 +111,10 @@ jobs:
         type: string
         default: "./..."
         description: Import paths of packages to be tested.
+      cwd:
+        type: string
+        default: "."
+        description: Directory in which to run the test command
       suite:
         type: string
         default: unit
@@ -110,9 +127,16 @@ jobs:
           command: make boostci
           no_output_timeout: 30m
       - download-params
+      # Note: setup_remote_docker takes about 10s and it's only needed for
+      # local index directory tests so it may be worth refactoring so that it's
+      # only included for local index directory
+      - setup_remote_docker:
+          version: 20.10.18
+          docker_layer_caching: true
       - run:
           name: go test
           command: |
+            cd << parameters.cwd >>
             go test \
               << parameters.go-test-flags >> \
               << parameters.target >>
@@ -306,7 +330,15 @@ workflows:
           suite: itest-markets_v1_offline_deal
           target: "./itests/markets_v1_offline_deal_test.go"
 
+      - test:
+          name: test-itest-disabled_markets_v1_deal
+          suite: itest-disabled_markets_v1_deal
+          target: "./itests/disabled_markets_v1_deal_test.go"
+
       - test:
           name: test-all
           suite: all
           target: "`go list ./... | grep -v boost/itests`"
+
+      - lid-docker-compose
+

.gitignore

@@ -1,12 +1,15 @@
 /boost
 /boostx
 /boostd
+/boostd-data
+/migrate-lid
 /devnet
 /booster-http
 /booster-bitswap
 /docgen-md
 /docgen-openrpc
 extern/filecoin-ffi/rust/target
+extern/boostd-data/boostd-data
 **/*.a
 **/*.pc
 /**/*/.DS_STORE

Makefile

@@ -7,6 +7,7 @@ unexport GOFLAGS
 
 GOCC?=go
 
+ARCH?=$(shell arch)
 GOVERSION:=$(shell $(GOCC) version | tr ' ' '\n' | grep go1 | sed 's/^go//' | awk -F. '{printf "%d%03d%03d", $$1, $$2, $$3}')
 ifeq ($(shell expr $(GOVERSION) \< 1016000), 1)
 $(warning Your Golang version is go$(shell expr $(GOVERSION) / 1000000).$(shell expr $(GOVERSION) % 1000000 / 1000).$(shell expr $(GOVERSION) % 1000))
@@ -79,19 +80,35 @@ calibnet-go: build-go
 deps: $(BUILD_DEPS)
 .PHONY: deps
 
+migrate-lid: $(BUILD_DEPS)
+	rm -f migrate-lid
+	$(GOCC) build $(GOFLAGS) -o migrate-lid ./cmd/migrate-lid
+.PHONY: migrate-lid
+BINS+=migrate-lid
+
 boostx: $(BUILD_DEPS)
 	rm -f boostx
 	$(GOCC) build $(GOFLAGS) -o boostx ./cmd/boostx
 .PHONY: boostx
 BINS+=boostx
 
 boost: $(BUILD_DEPS)
-	rm -f boost boostd boostx
+	rm -f boost
 	$(GOCC) build $(GOFLAGS) -o boost ./cmd/boost
-	$(GOCC) build $(GOFLAGS) -o boostx ./cmd/boostx
-	$(GOCC) build $(GOFLAGS) -o boostd ./cmd/boostd
 .PHONY: boost
-BINS+=boost boostx boostd
+BINS+=boost
+
+boostd: $(BUILD_DEPS)
+	rm -f boostd
+	$(GOCC) build $(GOFLAGS) -o boostd ./cmd/boostd
+.PHONY: boostd
+BINS+=boostd
+
+boostd-data:
+	$(MAKE) -C ./extern/boostd-data
+	install -C ./extern/boostd-data/boostd-data ./boostd-data
+.PHONY: boostd-data
+BINS+=boostd-data
 
 booster-http: $(BUILD_DEPS)
 	rm -f booster-http
@@ -126,7 +143,7 @@ update-react: validate-node-version
 	npm run --prefix react build
 .PHONY: react
 
-build-go: boost devnet
+build-go: boost boostd boostx boostd-data booster-http booster-bitswap devnet migrate-lid
 .PHONY: build-go
 
 build: react build-go
@@ -141,6 +158,10 @@ install-boost:
 	install -C ./boost /usr/local/bin/boost
 	install -C ./boostd /usr/local/bin/boostd
 	install -C ./boostx /usr/local/bin/boostx
+	install -C ./boostd-data /usr/local/bin/boostd-data
+	install -C ./booster-http /usr/local/bin/booster-http
+	install -C ./booster-bitswap /usr/local/bin/booster-bitswap
+	install -C ./migrate-lid /usr/local/bin/migrate-lid
 
 install-devnet:
 	install -C ./devnet /usr/local/bin/devnet
@@ -252,6 +273,9 @@ docker/all: $(lotus_build_cmd) docker/boost docker/booster-http docker/booster-b
 	docker/lotus docker/lotus-miner
 .PHONY: docker/all
 
+test-lid:
+	cd ./extern/boostd-data && ARCH=$(ARCH) docker-compose up --build --exit-code-from go-tests
+
 devnet/up:
 	rm -rf ./docker/devnet/data && docker compose -f ./docker/devnet/docker-compose.yaml up -d
 

api/api.go

@@ -3,7 +3,6 @@ package api
 import (
 	"context"
 
-	"github.com/filecoin-project/boost-gfm/piecestore"
 	"github.com/filecoin-project/boost-gfm/retrievalmarket"
 	"github.com/filecoin-project/boost-gfm/storagemarket"
 	smtypes "github.com/filecoin-project/boost/storagemarket/types"
@@ -35,6 +34,7 @@ type Boost interface {
 
 	// MethodGroup: Boost
 	BoostIndexerAnnounceAllDeals(ctx context.Context) error                                                                                     //perm:admin
+	BoostIndexerListMultihashes(ctx context.Context, proposalCid cid.Cid) ([]multihash.Multihash, error)                                        //perm:admin
 	BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error)                                                                            //perm:admin
 	BoostIndexerAnnounceLatestHttp(ctx context.Context, urls []string) (cid.Cid, error)                                                         //perm:admin
 	BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string, delAfterImport bool) (*ProviderDealRejectionInfo, error) //perm:admin
@@ -56,6 +56,9 @@ type Boost interface {
 	BlockstoreHas(ctx context.Context, c cid.Cid) (bool, error)    //perm:read
 	BlockstoreGetSize(ctx context.Context, c cid.Cid) (int, error) //perm:read
 
+	// MethodGroup: PieceDirectory
+	PdBuildIndexForPieceCid(ctx context.Context, piececid cid.Cid) error //perm:admin
+
 	// RuntimeSubsystems returns the subsystems that are enabled
 	// in this instance.
 	RuntimeSubsystems(ctx context.Context) (lapi.MinerSubsystems, error) //perm:read
@@ -75,12 +78,6 @@ type Boost interface {
 	MarketPendingDeals(ctx context.Context) (lapi.PendingDealInfo, error)                                                                                                                //perm:write
 	SectorsRefs(context.Context) (map[string][]lapi.SealedRef, error)                                                                                                                    //perm:read
 
-	PiecesListPieces(ctx context.Context) ([]cid.Cid, error)                                 //perm:read
-	PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error)                               //perm:read
-	PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) //perm:read
-	PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error)   //perm:read
-	PiecesGetMaxOffset(ctx context.Context, pieceCid cid.Cid) (uint64, error)                //perm:read
-
 	// MethodGroup: Actor
 	ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error) //perm:read
 

car/car_offset_writer.go

@@ -6,12 +6,12 @@ import (
 	"fmt"
 	"io"
 
-	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/boxo/blockservice"
+	blockstore "github.com/ipfs/boxo/blockstore"
+	offline "github.com/ipfs/boxo/exchange/offline"
+	"github.com/ipfs/boxo/ipld/merkledag"
 	"github.com/ipfs/go-cid"
-	blockstore "github.com/ipfs/go-ipfs-blockstore"
-	offline "github.com/ipfs/go-ipfs-exchange-offline"
 	format "github.com/ipfs/go-ipld-format"
-	"github.com/ipfs/go-merkledag"
 	"github.com/ipld/go-car"
 	"github.com/ipld/go-car/util"
 )

car/car_offset_writer_test.go

@@ -7,18 +7,18 @@ import (
 	"math/rand"
 	"testing"
 
-	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/boxo/blockservice"
+	bstore "github.com/ipfs/boxo/blockstore"
+	chunk "github.com/ipfs/boxo/chunker"
+	"github.com/ipfs/boxo/ipld/merkledag"
+	merkledagpb "github.com/ipfs/boxo/ipld/merkledag/pb"
+	"github.com/ipfs/boxo/ipld/unixfs/importer/balanced"
+	"github.com/ipfs/boxo/ipld/unixfs/importer/helpers"
+	blocks "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cidutil"
 	"github.com/ipfs/go-datastore"
 	dss "github.com/ipfs/go-datastore/sync"
-	bstore "github.com/ipfs/go-ipfs-blockstore"
-	chunk "github.com/ipfs/go-ipfs-chunker"
 	format "github.com/ipfs/go-ipld-format"
-	"github.com/ipfs/go-libipfs/blocks"
-	"github.com/ipfs/go-merkledag"
-	merkledagpb "github.com/ipfs/go-merkledag/pb"
-	"github.com/ipfs/go-unixfs/importer/balanced"
-	"github.com/ipfs/go-unixfs/importer/helpers"
 	"github.com/ipld/go-car"
 	mh "github.com/multiformats/go-multihash"
 	"github.com/stretchr/testify/require"

car/car_reader_seeker_test.go

@@ -8,11 +8,11 @@ import (
 	"math/rand"
 	"testing"
 
-	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/boxo/blockservice"
+	bstore "github.com/ipfs/boxo/blockstore"
+	"github.com/ipfs/boxo/ipld/merkledag"
 	"github.com/ipfs/go-datastore"
 	dss "github.com/ipfs/go-datastore/sync"
-	bstore "github.com/ipfs/go-ipfs-blockstore"
-	"github.com/ipfs/go-merkledag"
 	"github.com/stretchr/testify/require"
 )