Merge releases into v1.10 release

.circleci/config.yml

@@ -444,7 +444,7 @@ jobs:
       - run: make deps
       - run: go install golang.org/x/tools/cmd/goimports
       - run: go install github.com/hannahhoward/cbor-gen-for
-      - run: go generate ./...
+      - run: make type-gen
       - run: git --no-pager diff
       - run: git --no-pager diff --quiet
 
@@ -453,6 +453,7 @@ jobs:
     steps:
       - install-deps
       - prepare
+      - run: go install golang.org/x/tools/cmd/goimports
       - run: zcat build/openrpc/full.json.gz | jq > ../pre-openrpc-full
       - run: zcat build/openrpc/miner.json.gz | jq > ../pre-openrpc-miner
       - run: zcat build/openrpc/worker.json.gz | jq > ../pre-openrpc-worker

.github/CODEOWNERS

@@ -8,9 +8,9 @@
 ## the PR before merging.
 
 ### Global owners.
-*                       @magik6k @whyrusleeping @Kubuxu
+*                       @magik6k @arajasek
 
 ### Conformance testing.
-conformance/            @raulk
-extern/test-vectors     @raulk
-cmd/tvx                 @raulk
+conformance/            @ZenGround0
+extern/test-vectors     @ZenGround0
+cmd/tvx                 @ZenGround0

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,65 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

Makefile

@@ -233,6 +233,13 @@ testground:
 .PHONY: testground
 BINS+=testground
 
+
+tvx:
+	rm -f tvx
+	go build -o tvx ./cmd/tvx
+.PHONY: tvx
+BINS+=tvx
+
 install-chainwatch: lotus-chainwatch
 	install -C ./lotus-chainwatch /usr/local/bin/lotus-chainwatch
 
@@ -318,47 +325,49 @@ dist-clean:
 	git submodule deinit --all -f
 .PHONY: dist-clean
 
-type-gen:
+type-gen: api-gen
 	go run ./gen/main.go
-	go generate ./...
+	go generate -x ./...
+	goimports -w api/
 
-method-gen:
+method-gen: api-gen
 	(cd ./lotuspond/front/src/chain && go run ./methodgen.go)
 
 actors-gen:
 	go run ./chain/actors/agen
 	go fmt ./...
 
 api-gen:
-	go run ./gen/api > api/apistruct/struct.go
-	goimports -w api/apistruct
-	goimports -w api/apistruct
+	go run ./gen/api
+	goimports -w api
+	goimports -w api
 .PHONY: api-gen
 
 docsgen: docsgen-md docsgen-openrpc
 
-docsgen-md-bin: actors-gen
+docsgen-md-bin: api-gen actors-gen
 	go build $(GOFLAGS) -o docgen-md ./api/docgen/cmd
-docsgen-openrpc-bin: actors-gen
+docsgen-openrpc-bin: api-gen actors-gen
 	go build $(GOFLAGS) -o docgen-openrpc ./api/docgen-openrpc/cmd
 
 docsgen-md: docsgen-md-full docsgen-md-storage docsgen-md-worker
 
 docsgen-md-full: docsgen-md-bin
-	./docgen-md "api/api_full.go" "FullNode" > documentation/en/api-methods.md
+	./docgen-md "api/api_full.go" "FullNode" "api" "./api" > documentation/en/api-v1-unstable-methods.md
+	./docgen-md "api/v0api/full.go" "FullNode" "v0api" "./api/v0api" > documentation/en/api-v0-methods.md
 docsgen-md-storage: docsgen-md-bin
-	./docgen-md "api/api_storage.go" "StorageMiner" > documentation/en/api-methods-miner.md
+	./docgen-md "api/api_storage.go" "StorageMiner" "api" "./api" > documentation/en/api-v0-methods-miner.md
 docsgen-md-worker: docsgen-md-bin
-	./docgen-md "api/api_worker.go" "Worker" > documentation/en/api-methods-worker.md
+	./docgen-md "api/api_worker.go" "Worker" "api" "./api" > documentation/en/api-v0-methods-worker.md
 
 docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker
 
 docsgen-openrpc-full: docsgen-openrpc-bin
-	./docgen-openrpc "api/api_full.go" "FullNode" -gzip > build/openrpc/full.json.gz
+	./docgen-openrpc "api/api_full.go" "FullNode" "api" "./api" -gzip > build/openrpc/full.json.gz
 docsgen-openrpc-storage: docsgen-openrpc-bin
-	./docgen-openrpc "api/api_storage.go" "StorageMiner" -gzip > build/openrpc/miner.json.gz
+	./docgen-openrpc "api/api_storage.go" "StorageMiner" "api" "./api" -gzip > build/openrpc/miner.json.gz
 docsgen-openrpc-worker: docsgen-openrpc-bin
-	./docgen-openrpc "api/api_worker.go" "Worker" -gzip > build/openrpc/worker.json.gz
+	./docgen-openrpc "api/api_worker.go" "Worker" "api" "./api" -gzip > build/openrpc/worker.json.gz
 
 .PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin
 

api/README.md

@@ -0,0 +1,14 @@
+## Lotus API
+
+This package contains all lotus API definitions. Interfaces defined here are
+exposed as JsonRPC 2.0 endpoints by lotus programs.
+
+### Versions
+
+| File             | Alias File        | Interface      | Exposed by         | Version | HTTP Endpoint | Status                       | Docs
+|------------------|-------------------|----------------|--------------------|---------|---------------|------------------------------|------
+| `api_common.go`  | `v0api/latest.go` | `Common`       | lotus; lotus-miner | v0      | `/rpc/v0`     | Latest, Stable               | [Methods](../documentation/en/api-v0-methods.md)
+| `api_full.go`    | `v1api/latest.go` | `FullNode`     | lotus              | v1      | `/rpc/v1`     | Latest, **Work in progress** | [Methods](../documentation/en/api-v1-unstable-methods.md)
+| `api_storage.go` | `v0api/latest.go` | `StorageMiner` | lotus-miner        | v0      | `/rpc/v0`     | Latest, Stable               | [Methods](../documentation/en/api-v0-methods-miner.md)
+| `api_worker.go`  | `v0api/latest.go` | `Worker`       | lotus-worker       | v0      | `/rpc/v0`     | Latest, Stable               | [Methods](../documentation/en/api-v0-methods-worker.md)
+| `v0api/full.go`  |                   | `FullNode`     | lotus              | v0      | `/rpc/v0`     | Stable                       | [Methods](../documentation/en/api-v0-methods.md)

api/api_common.go

@@ -15,6 +15,17 @@ import (
 	apitypes "github.com/filecoin-project/lotus/api/types"
 )
 
+//                       MODIFYING THE API INTERFACE
+//
+// When adding / changing methods in this file:
+// * Do the change here
+// * Adjust implementation in `node/impl/`
+// * Run `make gen` - this will:
+//  * Generate proxy structs
+//  * Generate mocks
+//  * Generate markdown docs
+//  * Generate openrpc blobs
+
 type Common interface {
 
 	// MethodGroup: Auth

api/api_full.go

@@ -39,6 +39,22 @@ type ChainIO interface {
 	ChainHasObj(context.Context, cid.Cid) (bool, error)
 }
 
+const LookbackNoLimit = abi.ChainEpoch(-1)
+
+//                       MODIFYING THE API INTERFACE
+//
+// NOTE: This is the V1 (Unstable) API - to add methods to the V0 (Stable) API
+// you'll have to add those methods to interfaces in `api/v0api`
+//
+// When adding / changing methods in this file:
+// * Do the change here
+// * Adjust implementation in `node/impl/`
+// * Run `make gen` - this will:
+//  * Generate proxy structs
+//  * Generate mocks
+//  * Generate markdown docs
+//  * Generate openrpc blobs
+
 // FullNode API is a low-level interface to the Filecoin network full node
 type FullNode interface {
 	Common
@@ -430,56 +446,24 @@ type FullNode interface {
 	StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorExpiration, error) //perm:read
 	// StateSectorPartition finds deadline/partition with the specified sector
 	StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error) //perm:read
-	// StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed
-	//
-	// NOTE: If a replacing message is found on chain, this method will return
-	// a MsgLookup for the replacing message - the MsgLookup.Message will be a different
-	// CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
-	// result of the execution of the replacing message.
-	//
-	// If the caller wants to ensure that exactly the requested message was executed,
-	// they MUST check that MsgLookup.Message is equal to the provided 'cid'.
-	// Without this check both the requested and original message may appear as
-	// successfully executed on-chain, which may look like a double-spend.
-	//
-	// A replacing message is a message with a different CID, any of Gas values, and
-	// different signature, but with all other parameters matching (source/destination,
-	// nonce, params, etc.)
-	StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error) //perm:read
-	// StateSearchMsgLimited looks back up to limit epochs in the chain for a message, and returns its receipt and the tipset where it was executed
+	// StateSearchMsg looks back up to limit epochs in the chain for a message, and returns its receipt and the tipset where it was executed
 	//
 	// NOTE: If a replacing message is found on chain, this method will return
 	// a MsgLookup for the replacing message - the MsgLookup.Message will be a different
 	// CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
 	// result of the execution of the replacing message.
 	//
 	// If the caller wants to ensure that exactly the requested message was executed,
-	// they MUST check that MsgLookup.Message is equal to the provided 'cid'.
-	// Without this check both the requested and original message may appear as
+	// they must check that MsgLookup.Message is equal to the provided 'cid', or set the
+	// `allowReplaced` parameter to false. Without this check, and with `allowReplaced`
+	// set to true, both the requested and original message may appear as
 	// successfully executed on-chain, which may look like a double-spend.
 	//
 	// A replacing message is a message with a different CID, any of Gas values, and
 	// different signature, but with all other parameters matching (source/destination,
 	// nonce, params, etc.)
-	StateSearchMsgLimited(ctx context.Context, msg cid.Cid, limit abi.ChainEpoch) (*MsgLookup, error) //perm:read
-	// StateWaitMsg looks back in the chain for a message. If not found, it blocks until the
-	// message arrives on chain, and gets to the indicated confidence depth.
-	//
-	// NOTE: If a replacing message is found on chain, this method will return
-	// a MsgLookup for the replacing message - the MsgLookup.Message will be a different
-	// CID than the one provided in the 'cid' param, MsgLookup.Receipt will contain the
-	// result of the execution of the replacing message.
-	//
-	// If the caller wants to ensure that exactly the requested message was executed,
-	// they MUST check that MsgLookup.Message is equal to the provided 'cid'.
-	// Without this check both the requested and original message may appear as
-	// successfully executed on-chain, which may look like a double-spend.
-	//
-	// A replacing message is a message with a different CID, any of Gas values, and
-	// different signature, but with all other parameters matching (source/destination,
-	// nonce, params, etc.)
-	StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*MsgLookup, error) //perm:read
-	// StateWaitMsgLimited looks back up to limit epochs in the chain for a message.
+	StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error) //perm:read
+	// StateWaitMsg looks back up to limit epochs in the chain for a message.
 	// If not found, it blocks until the message arrives on chain, and gets to the
 	// indicated confidence depth.
 	//
@@ -489,14 +473,15 @@ type FullNode interface {
 	// result of the execution of the replacing message.
 	//
 	// If the caller wants to ensure that exactly the requested message was executed,
-	// they MUST check that MsgLookup.Message is equal to the provided 'cid'.
-	// Without this check both the requested and original message may appear as
+	// they must check that MsgLookup.Message is equal to the provided 'cid', or set the
+	// `allowReplaced` parameter to false. Without this check, and with `allowReplaced`
+	// set to true, both the requested and original message may appear as
 	// successfully executed on-chain, which may look like a double-spend.
 	//
 	// A replacing message is a message with a different CID, any of Gas values, and
 	// different signature, but with all other parameters matching (source/destination,
 	// nonce, params, etc.)
-	StateWaitMsgLimited(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch) (*MsgLookup, error) //perm:read
+	StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error) //perm:read
 	// StateListMiners returns the addresses of every miner that has claimed power in the Power Actor
 	StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error) //perm:read
 	// StateListActors returns the addresses of every actor in the state
@@ -516,16 +501,6 @@ type FullNode interface {
 	// StateChangedActors returns all the actors whose states change between the two given state CIDs
 	// TODO: Should this take tipset keys instead?
 	StateChangedActors(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) //perm:read
-	// StateGetReceipt returns the message receipt for the given message or for a
-	// matching gas-repriced replacing message
-	//
-	// NOTE: If the requested message was replaced, this method will return the receipt
-	// for the replacing message - if the caller needs the receipt for exactly the
-	// requested message, use StateSearchMsg().Receipt, and check that MsgLookup.Message
-	// is matching the requested CID
-	//
-	// DEPRECATED: Use StateSearchMsg, this method won't be supported in v1 API
-	StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error) //perm:read
 	// StateMinerSectorCount returns the number of sectors in a miner's sector set and proving set
 	StateMinerSectorCount(context.Context, address.Address, types.TipSetKey) (MinerSectors, error) //perm:read
 	// StateCompute is a flexible command that applies the given messages on the given tipset.
@@ -1038,11 +1013,12 @@ type DealCollateralBounds struct {
 }
 
 type CirculatingSupply struct {
-	FilVested      abi.TokenAmount
-	FilMined       abi.TokenAmount
-	FilBurnt       abi.TokenAmount
-	FilLocked      abi.TokenAmount
-	FilCirculating abi.TokenAmount
+	FilVested           abi.TokenAmount
+	FilMined            abi.TokenAmount
+	FilBurnt            abi.TokenAmount
+	FilLocked           abi.TokenAmount
+	FilCirculating      abi.TokenAmount
+	FilReserveDisbursed abi.TokenAmount
 }
 
 type MiningBaseInfo struct {

api/api_gateway.go

@@ -14,6 +14,20 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
+//                       MODIFYING THE API INTERFACE
+//
+// NOTE: This is the V1 (Unstable) API - to add methods to the V0 (Stable) API
+// you'll have to add those methods to interfaces in `api/v0api`
+//
+// When adding / changing methods in this file:
+// * Do the change here
+// * Adjust implementation in `node/impl/`
+// * Run `make gen` - this will:
+//  * Generate proxy structs
+//  * Generate mocks
+//  * Generate markdown docs
+//  * Generate openrpc blobs
+
 type Gateway interface {
 	ChainHasObj(context.Context, cid.Cid) (bool, error)
 	ChainHead(ctx context.Context) (*types.TipSet, error)
@@ -31,7 +45,6 @@ type Gateway interface {
 	StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
 	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
 	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
-	StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)
 	StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
 	StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
 	StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MarketBalance, error)
@@ -40,8 +53,9 @@ type Gateway interface {
 	StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error)
 	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*MinerPower, error)
 	StateNetworkVersion(context.Context, types.TipSetKey) (apitypes.NetworkVersion, error)
-	StateSearchMsg(ctx context.Context, msg cid.Cid) (*MsgLookup, error)
 	StateSectorGetInfo(ctx context.Context, maddr address.Address, n abi.SectorNumber, tsk types.TipSetKey) (*miner.SectorOnChainInfo, error)
 	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
-	StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*MsgLookup, error)
+	StateSearchMsg(ctx context.Context, from types.TipSetKey, msg cid.Cid, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
+	StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64, limit abi.ChainEpoch, allowReplaced bool) (*MsgLookup, error)
+	WalletBalance(context.Context, address.Address) (types.BigInt, error)
 }