Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test: DDO onboarding non-market verified data #11603

Merged
merged 7 commits into from
Feb 7, 2024
Merged

test: DDO onboarding non-market verified data #11603

Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7b6d763
test: DDO onboarding non-market verified data
rvagg Jan 30, 2024
ff17d7f
TEMP: extract events
rvagg Feb 1, 2024
b54d111
Extract buildActorEventsFromMessages and separate event printing code
rvagg Feb 2, 2024
3f2c3a3
solve actor resolution bug
aarshkshah1992 Feb 5, 2024
cc04504
Merge remote-tracking branch 'origin/feat/built-in-actor-events-api' …
aarshkshah1992 Feb 6, 2024
a694690
tests for events API
aarshkshah1992 Feb 7, 2024
f9c891b
lint
aarshkshah1992 Feb 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
320 changes: 320 additions & 0 deletions itests/direct_data_onboard_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,41 @@ import (
"bytes"
"context"
"crypto/rand"
"fmt"
"os"
"testing"
"time"

"github.com/ipfs/go-cid"
"github.com/ipld/go-ipld-prime"
"github.com/ipld/go-ipld-prime/codec/dagcbor"
"github.com/ipld/go-ipld-prime/codec/dagjson"
"github.com/stretchr/testify/require"

"github.com/filecoin-project/go-address"
cborutil "github.com/filecoin-project/go-cbor-util"
"github.com/filecoin-project/go-commp-utils/nonffi"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/builtin"
minertypes13 "github.com/filecoin-project/go-state-types/builtin/v13/miner"
verifregtypes13 "github.com/filecoin-project/go-state-types/builtin/v13/verifreg"
datacap2 "github.com/filecoin-project/go-state-types/builtin/v9/datacap"
market2 "github.com/filecoin-project/go-state-types/builtin/v9/market"
"github.com/filecoin-project/go-state-types/exitcode"
"github.com/filecoin-project/go-state-types/network"

"github.com/filecoin-project/lotus/api"
lapi "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/actors"
"github.com/filecoin-project/lotus/chain/actors/builtin/datacap"
"github.com/filecoin-project/lotus/chain/actors/builtin/market"
minertypes "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/actors/builtin/verifreg"
"github.com/filecoin-project/lotus/chain/consensus/filcns"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet/key"
"github.com/filecoin-project/lotus/itests/kit"
"github.com/filecoin-project/lotus/lib/must"
"github.com/filecoin-project/lotus/node/config"
Expand Down Expand Up @@ -102,6 +118,310 @@ func TestOnboardRawPiece(t *testing.T) {
require.Equal(t, dc.PieceCID, *si.CommD)
}

func TestOnboardRawPieceVerified(t *testing.T) {
kit.QuietMiningLogs()

var (
blocktime = 2 * time.Millisecond
ctx = context.Background()
)

rootKey, err := key.GenerateKey(types.KTSecp256k1)
require.NoError(t, err)

verifier1Key, err := key.GenerateKey(types.KTSecp256k1)
require.NoError(t, err)

verifiedClientKey, err := key.GenerateKey(types.KTBLS)
require.NoError(t, err)

bal, err := types.ParseFIL("100fil")
require.NoError(t, err)

client, miner, ens := kit.EnsembleMinimal(t, kit.ThroughRPC(),
kit.RootVerifier(rootKey, abi.NewTokenAmount(bal.Int64())),
kit.Account(verifier1Key, abi.NewTokenAmount(bal.Int64())),
kit.Account(verifiedClientKey, abi.NewTokenAmount(bal.Int64())),
)

evtChan, err := miner.FullNode.SubscribeActorEvents(ctx, &types.SubActorEventFilter{
Filter: types.ActorEventFilter{
MinEpoch: -1,
MaxEpoch: -1,
},
Prefill: true,
})
require.NoError(t, err)

events := make([]types.ActorEvent, 0)
go func() {
for e := range evtChan {
fmt.Printf("%s Got ActorEvent: %+v", time.Now().Format(time.StampMilli), e)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

currently doesn't show anything because they get filtered out due to address translation in the event filter

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed

events = append(events, *e)
}
}()

ens.InterconnectAll().BeginMiningMustPost(blocktime)

miner.PledgeSectors(ctx, 1, 0, nil)
sl, err := miner.SectorsListNonGenesis(ctx)
require.NoError(t, err)
require.Len(t, sl, 1, "expected 1 sector")

snum := sl[0]

maddr, err := miner.ActorAddress(ctx)
require.NoError(t, err)

client.WaitForSectorActive(ctx, t, snum, maddr)

pieceSize := abi.PaddedPieceSize(2048).Unpadded()
pieceData := make([]byte, pieceSize)
_, _ = rand.Read(pieceData)

dc, err := miner.ComputeDataCid(ctx, pieceSize, bytes.NewReader(pieceData))
require.NoError(t, err)

// get VRH
vrh, err := client.StateVerifiedRegistryRootKey(ctx, types.TipSetKey{})
fmt.Println(vrh.String())
require.NoError(t, err)

// import the root key.
rootAddr, err := client.WalletImport(ctx, &rootKey.KeyInfo)
require.NoError(t, err)

// import the verifiers' keys.
verifier1Addr, err := client.WalletImport(ctx, &verifier1Key.KeyInfo)
require.NoError(t, err)

// import the verified client's key.
verifiedClientAddr, err := client.WalletImport(ctx, &verifiedClientKey.KeyInfo)
require.NoError(t, err)

// make the 2 verifiers

mkVerifier(ctx, t, client.FullNode.(*api.FullNodeStruct), rootAddr, verifier1Addr)

// assign datacap to a client
initialDatacap := big.NewInt(10000)

params, err := actors.SerializeParams(&verifregtypes13.AddVerifiedClientParams{Address: verifiedClientAddr, Allowance: initialDatacap})
require.NoError(t, err)

msg := &types.Message{
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we really need to extract some of this logic into a helper to be reused by multiple tests

From: verifier1Addr,
To: verifreg.Address,
Method: verifreg.Methods.AddVerifiedClient,
Params: params,
Value: big.Zero(),
}

sm, err := client.MpoolPushMessage(ctx, msg, nil)
require.NoError(t, err)

res, err := client.StateWaitMsg(ctx, sm.Cid(), 1, lapi.LookbackNoLimit, true)
require.NoError(t, err)
require.EqualValues(t, 0, res.Receipt.ExitCode)

minerId, err := address.IDFromAddress(miner.ActorAddr)
require.NoError(t, err)

allocationRequest := verifregtypes13.AllocationRequest{
Provider: abi.ActorID(minerId),
Data: dc.PieceCID,
Size: dc.Size,
TermMin: verifregtypes13.MinimumVerifiedAllocationTerm,
TermMax: verifregtypes13.MaximumVerifiedAllocationTerm,
Expiration: verifregtypes13.MaximumVerifiedAllocationExpiration,
}

allocationRequests := verifregtypes13.AllocationRequests{
Allocations: []verifregtypes13.AllocationRequest{allocationRequest},
}

receiverParams, err := actors.SerializeParams(&allocationRequests)
require.NoError(t, err)

transferParams, err := actors.SerializeParams(&datacap2.TransferParams{
To: builtin.VerifiedRegistryActorAddr,
Amount: big.Mul(big.NewInt(int64(dc.Size)), builtin.TokenPrecision),
OperatorData: receiverParams,
})
require.NoError(t, err)

msg = &types.Message{
To: builtin.DatacapActorAddr,
From: verifiedClientAddr,
Method: datacap.Methods.TransferExported,
Params: transferParams,
Value: big.Zero(),
}

sm, err = client.MpoolPushMessage(ctx, msg, nil)
require.NoError(t, err)

res, err = client.StateWaitMsg(ctx, sm.Cid(), 1, lapi.LookbackNoLimit, true)
require.NoError(t, err)
require.EqualValues(t, 0, res.Receipt.ExitCode)

allocations, err := client.StateGetAllocations(ctx, verifiedClientAddr, types.EmptyTSK)
require.NoError(t, err)

require.Equal(t, 1, len(allocations))

var allocationId verifregtypes13.AllocationId
var clientId abi.ActorID
for key, value := range allocations {
allocationId = verifregtypes13.AllocationId(key)
clientId = value.Client
break
}

head, err := client.ChainHead(ctx)
require.NoError(t, err)

so, err := miner.SectorAddPieceToAny(ctx, pieceSize, bytes.NewReader(pieceData), piece.PieceDealInfo{
PublishCid: nil,
DealID: 0,
DealProposal: nil,
DealSchedule: piece.DealSchedule{
StartEpoch: head.Height() + 2880*2,
EndEpoch: head.Height() + 2880*400,
},
KeepUnsealed: true,
PieceActivationManifest: &minertypes.PieceActivationManifest{
CID: dc.PieceCID,
Size: dc.Size,
VerifiedAllocationKey: &minertypes13.VerifiedAllocationKey{Client: clientId, ID: allocationId},
Notify: nil,
},
})
require.NoError(t, err)

// wait for sector to commit
miner.WaitSectorsProving(ctx, map[abi.SectorNumber]struct{}{
so.Sector: {},
})

si, err := miner.SectorsStatus(ctx, so.Sector, true)
require.NoError(t, err)
require.Equal(t, dc.PieceCID, *si.CommD)

require.Equal(t, si.DealWeight, big.Zero())
require.Equal(t, si.VerifiedDealWeight, big.Mul(big.NewInt(int64(dc.Size)), big.NewInt(int64(si.Expiration-si.Activation))))

allocations, err = client.StateGetAllocations(ctx, verifiedClientAddr, types.EmptyTSK)
require.NoError(t, err)
require.Len(t, allocations, 0)

evts, err := miner.FullNode.GetActorEvents(ctx, &types.ActorEventFilter{
MinEpoch: -1,
MaxEpoch: -1,
})
require.NoError(t, err)

for _, evt := range evts {
fmt.Printf("Got ActorEvent: %+v", evt)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

also doesn't show anything for the same reason

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is fixed.

}

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

from here below we use the existing lotus APIs to get receipts and their event logs for the whole chain, which works

blockOutFile, err := os.Create("block.out")
require.NoError(t, err)
write := func(s string) {
_, err := blockOutFile.WriteString(s)
require.NoError(t, err)
}

head, err = miner.FullNode.ChainHead(ctx)
require.NoError(t, err)
for height := 0; height < int(head.Height()); height++ {
// for each tipset
ts, err := miner.FullNode.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(height), types.EmptyTSK)
require.NoError(t, err)
for _, b := range ts.Blocks() {
if b.Miner == miner.ActorAddr {
// for each block
// alternative here is to go straight to receipts: miner.FullNode.ChainGetParentReceipts(ctx, b.Cid())
messages, err := miner.FullNode.ChainGetParentMessages(ctx, b.Cid())
require.NoError(t, err)
if len(messages) == 0 {
continue
}
write(fmt.Sprintf("Height=%d Block=%s:", height, b.Cid()))
for _, parent := range b.Parents {
write(fmt.Sprintf(" %s", parent))
}
write("\n")
for _, m := range messages {
// for each message
write(fmt.Sprintf(" Message=%s: %s -> %s, %d\n", m.Cid, m.Message.From, m.Message.To, m.Message.Method))
receipt, err := miner.FullNode.StateSearchMsg(ctx, ts.Key(), m.Cid, -1, false)
require.NoError(t, err)
// receipt
write(fmt.Sprintf(" Receipt Exit=%d, Gas=%d, Return=0x%x\n", receipt.Receipt.ExitCode, receipt.Receipt.GasUsed, receipt.Receipt.Return))
if receipt.Receipt.EventsRoot == nil {
write(fmt.Sprintln(" No events"))
} else {
// receipt
events, err := miner.FullNode.ChainGetEvents(ctx, *receipt.Receipt.EventsRoot)
require.NoError(t, err)
for ii, evt := range events {
// for each event
addr, err := address.NewIDAddress(uint64(evt.Emitter))
require.NoError(t, err)
write(fmt.Sprintf(" Event=%d (%s):\n", ii, addr))
for _, e := range evt.Entries {
// for each event entry
write(fmt.Sprintf(" Entry=0x%x, 0x%x, %s=%s\n", e.Codec, e.Flags, e.Key, toDagJson(t, e.Codec, e.Value)))
}
}
}
}
}
}
}
require.NoError(t, blockOutFile.Close())
fmt.Println("Wrote block.out")
}

func toDagJson(t *testing.T, codec uint64, data []byte) string {
switch codec {
case 0x51:
nd, err := ipld.Decode(data, dagcbor.Decode)
require.NoError(t, err)
byts, err := ipld.Encode(nd, dagjson.Encode)
require.NoError(t, err)
return string(byts)
default:
return fmt.Sprintf("0x%x", data)
}
}

func mkVerifier(ctx context.Context, t *testing.T, api *api.FullNodeStruct, rootAddr address.Address, addr address.Address) {
allowance := big.NewInt(100000000000)
params, aerr := actors.SerializeParams(&verifregtypes13.AddVerifierParams{Address: addr, Allowance: allowance})
require.NoError(t, aerr)

msg := &types.Message{
From: rootAddr,
To: verifreg.Address,
Method: verifreg.Methods.AddVerifier,
Params: params,
Value: big.Zero(),
}

sm, err := api.MpoolPushMessage(ctx, msg, nil)
require.NoError(t, err, "AddVerifier failed")

res, err := api.StateWaitMsg(ctx, sm.Cid(), 1, lapi.LookbackNoLimit, true)
require.NoError(t, err)
require.EqualValues(t, 0, res.Receipt.ExitCode)

verifierAllowance, err := api.StateVerifierStatus(ctx, addr, types.EmptyTSK)
require.NoError(t, err)
require.Equal(t, allowance, *verifierAllowance)
}

func makeMarketDealProposal(t *testing.T, client *kit.TestFullNode, miner *kit.TestMiner, data cid.Cid, ps abi.PaddedPieceSize, start, end abi.ChainEpoch) market2.ClientDealProposal {
ca, err := client.WalletDefaultAddress(context.Background())
require.NoError(t, err)
Expand Down