Skip to content

Commit

Permalink
merge: main to lid (#1370)
Browse files Browse the repository at this point in the history 
* add free check (#1315)

* chore: bump version to 1.6.1 (#1317)

* fix legacy deal verified status (#1324)

* fix: update go-unixfsnode enough to make sure unixfs-preload is available (#1323)

* release v1.6.2-rc1 (#1328)

* use full path (#1330)

* fix bug (#1332)

* use forks of graphsync, go-data-transfer and go-fil-markets (#1333)

* refactor: use forks of graphsync, go-data-transfer and go-fil-markets

* refactor: convert from data transfer v1 to v2 voucher type

* fix: index provider validation voucher type

* fix: pass index provider engine link system through to graphsync's transport configurer

* feat: use tagged version of boost-gfm

* fix: retrieval client imports

* feat: tagged version of lotus

* feat: require go 1.19

* lint: fix lint errors

* fix: itests

* fix: cbor-gen, docsgen

* fix: update CI lint version

* fix: lint

* fix: docgen

* fix: go mod tidy

* fix: protocol proxy TestOutboundForwarding

* fix: docsgen

* fix: update filecoin-ffi submodule

* fix: prometheus duplicate register panic

* fix: cleanup imports

* fix: legs voucher processing

* chore: release v1.6.2-rc2 (#1340)

* release v1.6.2-rc2

* fix test

* fix: flaky TestLibp2pCarServerNewTransferCancelsPreviousTransfer (#1350)

* fix: flaky TestDealCompletionOnProcessResumption (#1351)

* fix: occasional panic on shutdown (#1353)

* feat: query UI (#1352)

* log insert

* fix display error

* refactor code

* shorten status strings

* remove comment

* apply suggestion

* feat: add download block link to inspect page (#1312)

* fix(devnet): update golang and lotus default versions (#1354)

* fix(devnet): bump golang to 1.19

* chore(devnet): bump lotus default version

* chore(devnet): remove unused stable env

* booster-http: implement IPFS HTTP gateway (#1225)

* feat: implement http api gateway

* feat: use go-libipfs lib (instead of copying to extern)

* feat: bump booster-bitswap info minor version

* feat: http gateway metrics

* fix: TestHttpInfo

* feat: by default only serve blocks and CARs, with option to serve original files (jpg, mov etc)

* fix: correct link for download root block (#1355)

* feat: option to cleanup data for offline deals after add piece (#1341)

* chore: add support for multiple node.js versions in makefile (#1356)

* chore: release v.1.7.0-rc1 (#1357)

* release v.1.7.0-rc1

* fix version

* fix: dagstore initialize-all parameter (#1363)

* fix: show verifying commp state for offline deals (#1364)

* fix: boost run missing staging-area dir (#1368)

* merge(wip): main to lid

TODO: remoteblockstore needs to handle nil metrics

* fix: flaky TestNewHttpServer (#1372)

* feat: group agent version by binary name (#1369)

* fix: wrap stats in nil checks for now

we should probably revisit how stats are handled now that we have all 3 transports being tracked

* test(fix): incorrect test urls

---------

Co-authored-by: LexLuthr <88259624+LexLuthr@users.noreply.github.com>
Co-authored-by: Rod Vagg <rod@vagg.org>
Co-authored-by: dirkmc <dirkmdev@gmail.com>
  • Loading branch information
@jacobheun @LexLuthr
@rvagg @dirkmc
4 people authored Apr 13, 2023
1 parent 2172161 commit d9923e6
Show file tree
Hide file tree
Showing 68 changed files with 2,027 additions and 303 deletions.
22 changes: 9 additions & 13 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,13 @@ $(warning Your Golang version is go$(shell expr $(GOVERSION) / 1000000).$(shell
$(error Update Golang to version to at least 1.16.0)
endif

LTS_NODE_VER=16
NODE_VER=$(shell node -v)
ifeq ($(patsubst v$(LTS_NODE_VER).%,matched,$(NODE_VER)), matched)
NODE_LTS=true
ALLOWED_NODE_VERSIONS := 16 18
validate-node-version:
ifeq ($(filter $(shell node -v | cut -c2-3),$(ALLOWED_NODE_VERSIONS)),)
@echo "Unsupported Node.js version. Please install one of the following versions: $(ALLOWED_NODE_VERSIONS)"
exit 1
else
NODE_LTS=false
@echo "Node.js version $(shell node -v) is supported."
endif

# git modules that need to be loaded
Expand Down Expand Up @@ -115,21 +116,16 @@ boostci: $(BUILD_DEPS)
$(GOCC) build $(GOFLAGS) -o boostci ./cmd/boostci
.PHONY: boostci

react: check-node-lts
react: validate-node-version
npm_config_legacy_peer_deps=yes npm ci --no-audit --prefix react
npm run --prefix react build
.PHONY: react

update-react: check-node-lts
update-react: validate-node-version
npm_config_legacy_peer_deps=yes npm install --no-audit --prefix react
npm run --prefix react build
.PHONY: react

.PHONY: check-node-lts
check-node-lts:
@$(NODE_LTS) || echo Build requires Node v$(LTS_NODE_VER) \(detected Node $(NODE_VER)\)
@$(NODE_LTS) && echo Building using Node v$(LTS_NODE_VER)

build-go: boost devnet
.PHONY: build-go

Expand Down Expand Up @@ -200,7 +196,7 @@ docsgen-openrpc-boost: docsgen-openrpc-bin

## DOCKER IMAGES
docker_user?=filecoin
lotus_version?=v1.20.0-rc1
lotus_version?=v1.21.0-rc2
ffi_from_source?=0
build_lotus?=0
ifeq ($(build_lotus),1)
Expand Down
30 changes: 15 additions & 15 deletions api/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,21 +33,21 @@ type Boost interface {
Net

// MethodGroup: Boost
BoostIndexerAnnounceAllDeals(ctx context.Context) error //perm:admin
BoostIndexerListMultihashes(ctx context.Context, proposalCid cid.Cid) ([]multihash.Multihash, error) //perm:admin
BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string) (*ProviderDealRejectionInfo, error) //perm:admin
BoostDeal(ctx context.Context, dealUuid uuid.UUID) (*smtypes.ProviderDealState, error) //perm:admin
BoostDealBySignedProposalCid(ctx context.Context, proposalCid cid.Cid) (*smtypes.ProviderDealState, error) //perm:admin
BoostDummyDeal(context.Context, smtypes.DealParams) (*ProviderDealRejectionInfo, error) //perm:admin
BoostDagstoreRegisterShard(ctx context.Context, key string) error //perm:admin
BoostDagstoreDestroyShard(ctx context.Context, key string) error //perm:admin
BoostDagstoreInitializeShard(ctx context.Context, key string) error //perm:admin
BoostDagstoreInitializeAll(ctx context.Context, params DagstoreInitializeAllParams) (<-chan DagstoreInitializeAllEvent, error) //perm:admin
BoostDagstoreRecoverShard(ctx context.Context, key string) error //perm:admin
BoostDagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin
BoostDagstorePiecesContainingMultihash(ctx context.Context, mh multihash.Multihash) ([]cid.Cid, error) //perm:read
BoostDagstoreListShards(ctx context.Context) ([]DagstoreShardInfo, error) //perm:admin
BoostMakeDeal(context.Context, smtypes.DealParams) (*ProviderDealRejectionInfo, error) //perm:write
BoostIndexerAnnounceAllDeals(ctx context.Context) error //perm:admin
BoostIndexerListMultihashes(ctx context.Context, proposalCid cid.Cid) ([]multihash.Multihash, error) //perm:admin
BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string, delAfterImport bool) (*ProviderDealRejectionInfo, error) //perm:admin
BoostDeal(ctx context.Context, dealUuid uuid.UUID) (*smtypes.ProviderDealState, error) //perm:admin
BoostDealBySignedProposalCid(ctx context.Context, proposalCid cid.Cid) (*smtypes.ProviderDealState, error) //perm:admin
BoostDummyDeal(context.Context, smtypes.DealParams) (*ProviderDealRejectionInfo, error) //perm:admin
BoostDagstoreRegisterShard(ctx context.Context, key string) error //perm:admin
BoostDagstoreDestroyShard(ctx context.Context, key string) error //perm:admin
BoostDagstoreInitializeShard(ctx context.Context, key string) error //perm:admin
BoostDagstoreInitializeAll(ctx context.Context, params DagstoreInitializeAllParams) (<-chan DagstoreInitializeAllEvent, error) //perm:admin
BoostDagstoreRecoverShard(ctx context.Context, key string) error //perm:admin
BoostDagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin
BoostDagstorePiecesContainingMultihash(ctx context.Context, mh multihash.Multihash) ([]cid.Cid, error) //perm:read
BoostDagstoreListShards(ctx context.Context) ([]DagstoreShardInfo, error) //perm:admin
BoostMakeDeal(context.Context, smtypes.DealParams) (*ProviderDealRejectionInfo, error) //perm:write

// MethodGroup: Blockstore
BlockstoreGet(ctx context.Context, c cid.Cid) ([]byte, error) //perm:read
Expand Down
8 changes: 4 additions & 4 deletions api/proxy_gen.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Binary file modified build/openrpc/boost.json.gz
View file
Binary file not shown.
2 changes: 1 addition & 1 deletion build/version.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package build

var CurrentCommit string

const BuildVersion = "1.6.2-rc1"
const BuildVersion = "1.7.0-rc1"

func UserVersion() string {
return BuildVersion + CurrentCommit
Expand Down
2 changes: 1 addition & 1 deletion cmd/boostd/dagstore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ var dagstoreInitializeAllCmd = &cli.Command{
},
Action: func(cctx *cli.Context) error {
concurrency := cctx.Uint("concurrency")
sealed := cctx.Bool("sealed")
sealed := cctx.Bool("include-sealed")

ctx := lcli.ReqContext(cctx)

Expand Down
15 changes: 14 additions & 1 deletion cmd/boostd/import_data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,13 @@ var importDataCmd = &cli.Command{
Name: "import-data",
Usage: "Import data for offline deal made with Boost",
ArgsUsage: "<proposal CID> <file> or <deal UUID> <file>",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "delete-after-import",
Usage: "whether to delete the data for the offline deal after the deal has been added to a sector",
Value: false,
},
},
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() < 2 {
return fmt.Errorf("must specify proposal CID / deal UUID and file path")
Expand Down Expand Up @@ -58,7 +65,12 @@ var importDataCmd = &cli.Command{
defer closer()

// If the user has supplied a signed proposal cid
deleteAfterImport := cctx.Bool("delete-after-import")
if proposalCid != nil {
if deleteAfterImport {
return fmt.Errorf("legacy deal data cannot be automatically deleted after import (only new deals)")
}

// Look up the deal in the boost database
deal, err := napi.BoostDealBySignedProposalCid(cctx.Context, *proposalCid)
if err != nil {
Expand All @@ -74,6 +86,7 @@ var importDataCmd = &cli.Command{
if err != nil {
return fmt.Errorf("couldnt import v1.1.0 deal, or find boost deal: %w", err)
}

fmt.Printf("Offline deal import for v1.1.0 deal %s scheduled for execution\n", proposalCid.String())
return nil
}
Expand All @@ -83,7 +96,7 @@ var importDataCmd = &cli.Command{
}

// Deal proposal by deal uuid (v1.2.0 deal)
rej, err := napi.BoostOfflineDealWithData(cctx.Context, dealUuid, filePath)
rej, err := napi.BoostOfflineDealWithData(cctx.Context, dealUuid, filePath, deleteAfterImport)
if err != nil {
return fmt.Errorf("failed to execute offline deal: %w", err)
}
Expand Down
20 changes: 17 additions & 3 deletions cmd/booster-bitswap/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ import (
"net/http"
_ "net/http/pprof"

"github.com/filecoin-project/boost/cmd/booster-bitswap/filters"
"github.com/filecoin-project/boost/cmd/booster-bitswap/remoteblockstore"
"github.com/filecoin-project/boost/cmd/lib"
"github.com/filecoin-project/boost/cmd/lib/filters"
"github.com/filecoin-project/boost/cmd/lib/remoteblockstore"
"github.com/filecoin-project/boost/metrics"
"github.com/filecoin-project/boost/piecedirectory"
"github.com/filecoin-project/boostd-data/shared/tracing"
Expand Down Expand Up @@ -165,6 +165,20 @@ var runCmd = &cli.Command{
}

// Create the bitswap host
bitswapBlockMetrics := remoteblockstore.BlockMetrics{
GetRequestCount: metrics.BitswapRblsGetRequestCount,
GetFailResponseCount: metrics.BitswapRblsGetFailResponseCount,
GetSuccessResponseCount: metrics.BitswapRblsGetSuccessResponseCount,
BytesSentCount: metrics.BitswapRblsBytesSentCount,
HasRequestCount: metrics.BitswapRblsHasRequestCount,
HasFailResponseCount: metrics.BitswapRblsHasFailResponseCount,
HasSuccessResponseCount: metrics.BitswapRblsHasSuccessResponseCount,
GetSizeRequestCount: metrics.BitswapRblsGetSizeRequestCount,
GetSizeFailResponseCount: metrics.BitswapRblsGetSizeFailResponseCount,
GetSizeSuccessResponseCount: metrics.BitswapRblsGetSizeSuccessResponseCount,
}

// Create the server API
port := cctx.Int("port")
repoDir, err := homedir.Expand(cctx.String(FlagRepo.Name))
if err != nil {
Expand All @@ -183,7 +197,7 @@ var runCmd = &cli.Command{
}
pr := &piecedirectory.SectorAccessorAsPieceReader{SectorAccessor: sa}
piecedirectory := piecedirectory.NewPieceDirectory(pdClient, pr, cctx.Int("add-index-throttle"))
remoteStore := remoteblockstore.NewRemoteBlockstore(piecedirectory)
remoteStore := remoteblockstore.NewRemoteBlockstore(piecedirectory, &bitswapBlockMetrics)
server := NewBitswapServer(remoteStore, host, multiFilter)

var proxyAddrInfo *peer.AddrInfo
Expand Down
Loading

0 comments on commit d9923e6

Please sign in to comment.