From 4502b99473a5b4029d168b8d2b3203acebeefaf2 Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 8 Nov 2023 19:05:28 +0800 Subject: [PATCH 01/14] fix --- ethstorage/node/node_mine_test.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/ethstorage/node/node_mine_test.go b/ethstorage/node/node_mine_test.go index ab849a48..0d0d9ec3 100644 --- a/ethstorage/node/node_mine_test.go +++ b/ethstorage/node/node_mine_test.go @@ -32,8 +32,7 @@ import ( const ( rpcUrl = "http://65.108.236.27:8545" - chainID = "7011893058" - kvEntriesPerShard = 16 + kvEntriesPerShard = 8192 kvSize = 128 * 1024 chunkSize = 128 * 1024 maxBlobsPerTx = 4 @@ -42,7 +41,7 @@ const ( var ( minerAddr = common.HexToAddress("0x04580493117292ba13361D8e9e28609ec112264D") - contractAddr = common.HexToAddress("0x188aac000e21ec314C5694bB82035b72210315A8") + contractAddr = common.HexToAddress("0xE31DbfB4d12B67eE60690Ad8a5877Ce8D77842ED") private = "95eb6ffd2ae0b115db4d1f0d58388216f9d026896696a5211d77b5f14eb5badf" shardIds = []uint64{0, 1} l = esLog.NewLogger(esLog.CLIConfig{ @@ -215,6 +214,12 @@ func prepareData(t *testing.T, n *EsNode, contract common.Address) { txs = txs + 1 } t.Logf("tx len %d \n", txs) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + chainID, err := n.l1Source.ChainID(ctx) + if err != nil { + t.Fatalf("Get chain id failed %v", err) + } for i := 0; i < txs; i++ { max := maxBlobsPerTx if i == txs-1 { @@ -228,15 +233,16 @@ func prepareData(t *testing.T, n *EsNode, contract common.Address) { if len(blobData) == 0 { break } - kvIdxes, dataHashes, err := utils.UploadBlobs(n.l1Source, rpcUrl, private, chainID, contract, blobData, false, value) + kvIdxes, dataHashes, err := utils.UploadBlobs(n.l1Source, rpcUrl, private, chainID.String(), contract, blobData, false, value) if err != nil { t.Fatalf("Upload blobs failed %v", err) } + t.Logf("kvIdxes=%v \n", kvIdxes) + t.Logf("dataHashes=%x \n", dataHashes) hashs = append(hashs, dataHashes...) ids = append(ids, kvIdxes...) - t.Logf("ids=%v \n", ids) } - block, err := n.l1Source.BlockNumber(context.Background()) + block, err := n.l1Source.BlockNumber(ctx) if err != nil { t.Fatalf("Failed to get block number %v", err) } From 947846ca85c0ae0e1e241d365e542fa12665b7e4 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 9 Nov 2023 12:01:17 +0800 Subject: [PATCH 02/14] update contract addr --- ethstorage/prover/kzg_prover_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ethstorage/prover/kzg_prover_test.go b/ethstorage/prover/kzg_prover_test.go index 29109dae..1c7aa7b4 100644 --- a/ethstorage/prover/kzg_prover_test.go +++ b/ethstorage/prover/kzg_prover_test.go @@ -26,9 +26,9 @@ import ( var ( rpc = "http://65.108.236.27:8545" - chainID = "7011893058" + chainID = "7011893061" prvk = "95eb6ffd2ae0b115db4d1f0d58388216f9d026896696a5211d77b5f14eb5badf" - contractAddr = common.HexToAddress("0xb55D964a4a9d73460393dbB0EB9b6565DB5c1362") + contractAddr = common.HexToAddress("0xE31DbfB4d12B67eE60690Ad8a5877Ce8D77842ED") value = hexutil.EncodeUint64(10000000000000) ) @@ -90,8 +90,8 @@ func uploadBlob(data []byte) common.Hash { -1, value, 510000, - "30000000000", - "20000000000", + "", + "", "10000000000", chainID, calldata, From d63ea9512480dd5694bc88417083a9c5bf5309f8 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 9 Nov 2023 15:22:21 +0800 Subject: [PATCH 03/14] remove testMine for repeated logic tested in node_mine_test.go --- ethstorage/miner/miner_test.go | 99 ++-------------------------------- 1 file changed, 4 insertions(+), 95 deletions(-) diff --git a/ethstorage/miner/miner_test.go b/ethstorage/miner/miner_test.go index 249365dc..1c24b64c 100644 --- a/ethstorage/miner/miner_test.go +++ b/ethstorage/miner/miner_test.go @@ -4,12 +4,7 @@ package miner import ( - "context" - "fmt" - "io" "math/big" - "net/http" - "os" "path/filepath" "runtime/debug" "testing" @@ -17,29 +12,20 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/event" - "github.com/ethstorage/go-ethstorage/cmd/es-utils/utils" es "github.com/ethstorage/go-ethstorage/ethstorage" "github.com/ethstorage/go-ethstorage/ethstorage/eth" esLog "github.com/ethstorage/go-ethstorage/ethstorage/log" "github.com/ethstorage/go-ethstorage/ethstorage/p2p/protocol" "github.com/ethstorage/go-ethstorage/ethstorage/prover" - "github.com/ethstorage/go-ethstorage/ethstorage/signer" - "golang.org/x/term" ) const ( - rpcUrl = "http://65.108.236.27:8545" - chainID = "7011893058" kvSizeBits = 17 - sampleLenBits = 12 kvEntriesBits = 1 - private = "95eb6ffd2ae0b115db4d1f0d58388216f9d026896696a5211d77b5f14eb5badf" ) var ( - // Note a new contract need to be deployed each time the test is being executed. contractAddr = common.HexToAddress("0x8FA1872c159DD8681119000d1C7a8Df52a8C128F") minerAddr = common.HexToAddress("0x04580493117292ba13361D8e9e28609ec112264D") fileName = "test_shard_0.dat" @@ -47,45 +33,10 @@ var ( kvEntries uint64 = 1 << kvEntriesBits shardID = uint64(0) value = hexutil.EncodeUint64(10000000000000) - lg = esLog.NewLogger(esLog.CLIConfig{ - Level: "debug", - Format: "text", - Color: term.IsTerminal(int(os.Stdout.Fd())), - }) + lg = esLog.NewLogger(esLog.DefaultCLIConfig()) ) -func TestMine(t *testing.T) { - client, err := eth.Dial(rpcUrl, contractAddr, lg) - if err != nil { - t.Fatalf("failed to get client L1: %v", err) - } - storageMgr := initStorageManager(t, client, true) - defer func() { - client.Close() - storageMgr.Close() - err := os.Remove(fileName) - if err != nil { - t.Errorf("cannot remove %s: %v", fileName, err) - } - }() - miner := newMiner(t, storageMgr, client) - eth.WatchHeadChanges(context.Background(), client, func(ctx context.Context, sig eth.L1BlockRef) { - select { - case miner.ChainHeadCh <- sig: - default: - } - }) - miner.Start() - miner.feed.Send(protocol.EthStorageSyncDone{ - DoneType: protocol.SingleShardDone, - ShardId: shardID, - }) - time.Sleep(300 * time.Second) - miner.Close() - checkMiningState(t, miner, false) -} - -func initStorageManager(t *testing.T, client *eth.PollingClient, withData bool) *es.StorageManager { +func initStorageManager(t *testing.T, client *eth.PollingClient) *es.StorageManager { df, err := es.Create(fileName, shardID, kvEntries, 0, kvSize, es.ENCODE_BLOB_POSEIDON, minerAddr, kvSize) if err != nil { t.Fatalf("Create failed %v", err) @@ -93,37 +44,10 @@ func initStorageManager(t *testing.T, client *eth.PollingClient, withData bool) shardMgr := es.NewShardManager(contractAddr, kvSize, kvEntries, kvSize) shardMgr.AddDataShard(shardID) shardMgr.AddDataFile(df) - if withData { - dataRaw, err := readFile() - if err != nil { - t.Fatalf("Read raw data failed %v", err) - } - kvIdxes, dataHashes, err := utils.UploadBlobs(client, rpcUrl, private, chainID, contractAddr, dataRaw, true, value) - if err != nil { - t.Fatalf("Upload blobs failed %v", err) - } - blobs := utils.EncodeBlobs(dataRaw) - for i, blob := range blobs { - kvIdx := kvIdxes[i] % kvEntries - t.Logf("TryWrite kvIdx: %d kvHash: %x\n", kvIdx, dataHashes[i]) - _, err := shardMgr.TryWrite(kvIdx, blob[:], dataHashes[i]) - if err != nil { - t.Fatalf("write failed: %v", err) - } - } - } return es.NewStorageManager(shardMgr, client) } func newMiner(t *testing.T, storageMgr *es.StorageManager, client *eth.PollingClient) *Miner { - key, _ := crypto.HexToECDSA(private) - signerAddr := crypto.PubkeyToAddress(key.PublicKey) - signerFnFactory, signerAddr, err := signer.SignerFactoryFromConfig( - signer.CLIConfig{}, - ) - if err != nil { - t.Fatalf("failed to get signer: %v", err) - } defaultConfig := &Config{ RandomChecks: 2, NonceLimit: 1048576, @@ -134,8 +58,6 @@ func newMiner(t *testing.T, storageMgr *es.StorageManager, client *eth.PollingCl PriorityGasPrice: new(big.Int).SetUint64(10), ThreadsPerShard: 1, ZKeyFileName: "blob_poseidon.zkey", - SignerFnFactory: signerFnFactory, - SignerAddr: signerAddr, } l1api := NewL1MiningAPI(client, lg) zkWorkingDir, _ := filepath.Abs("../prover") @@ -157,24 +79,10 @@ func checkMiningState(t *testing.T, m *Miner, mining bool) { t.Fatalf("Mining() == %t, want %t", state, mining) } -func readFile() ([]byte, error) { - txt_170k := "https://www.gutenberg.org/cache/epub/11/pg11.txt" - resp, err := http.Get(txt_170k) - if err != nil { - return nil, fmt.Errorf("error reading blob txtUrl: %v", err) - } - defer resp.Body.Close() - data, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("error reading blob txtUrl: %v", err) - } - return data, nil -} - func TestMiner_update(t *testing.T) { var shard = []uint64{0, 1, 2} // Case: happy path - storageMgr := initStorageManager(t, nil, false) + storageMgr := initStorageManager(t, nil) miner := newMiner(t, storageMgr, nil) miner.Start() // waiting for sync done @@ -221,4 +129,5 @@ func TestMiner_update(t *testing.T) { } miner.Close() checkMiningState(t, miner, false) + storageMgr.Close() } From 0a55478e25080685660225ae79ad77b9ee3004f0 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 9 Nov 2023 18:42:12 +0800 Subject: [PATCH 04/14] integration tests --- ethstorage/miner/miner_test.go | 2 + ethstorage/prover/zk_prover.go | 7 +- integration_tests/common.go | 67 +++++++ .../kzg_poseidon_prover_test.go | 5 +- .../kzg_prover_test.go | 90 +++------- .../node_mine_test.go | 170 +++++++++++------- .../signature_test.go | 32 ++-- .../zk_prover_test.go | 49 ++--- 8 files changed, 253 insertions(+), 169 deletions(-) create mode 100644 integration_tests/common.go rename {ethstorage/prover => integration_tests}/kzg_poseidon_prover_test.go (87%) rename {ethstorage/prover => integration_tests}/kzg_prover_test.go (67%) rename {ethstorage/node => integration_tests}/node_mine_test.go (60%) rename {ethstorage/signer => integration_tests}/signature_test.go (82%) rename {ethstorage/prover => integration_tests}/zk_prover_test.go (81%) diff --git a/ethstorage/miner/miner_test.go b/ethstorage/miner/miner_test.go index 1c24b64c..73cb7529 100644 --- a/ethstorage/miner/miner_test.go +++ b/ethstorage/miner/miner_test.go @@ -5,6 +5,7 @@ package miner import ( "math/big" + "os" "path/filepath" "runtime/debug" "testing" @@ -130,4 +131,5 @@ func TestMiner_update(t *testing.T) { miner.Close() checkMiningState(t, miner, false) storageMgr.Close() + os.Remove(fileName) } diff --git a/ethstorage/prover/zk_prover.go b/ethstorage/prover/zk_prover.go index d16f6d50..8265be35 100644 --- a/ethstorage/prover/zk_prover.go +++ b/ethstorage/prover/zk_prover.go @@ -42,6 +42,10 @@ func NewZKProver(workingDir, zkeyFile string, lg log.Logger) *ZKProver { return newZKProver(workingDir, zkeyFile, true, lg) } +func NewZKProverInternal(workingDir, zkeyFile string, lg log.Logger) *ZKProver { + return newZKProver(workingDir, zkeyFile, false, lg) +} + func newZKProver(workingDir, zkeyFile string, cleanup bool, lg log.Logger) *ZKProver { path := workingDir if path == "" { @@ -81,7 +85,8 @@ func (p *ZKProver) GenerateZKProof(encodingKey common.Hash, sampleIdx uint64) (Z } err := os.Mkdir(buildDir, os.ModePerm) if err != nil { - p.lg.Crit("Generate zk proof failed", "mkdir", buildDir, "error", err) + p.lg.Error("Generate zk proof failed", "mkdir", buildDir, "error", err) + return ZKProof{}, common.Hash{}, err } defer func() { if p.cleanup { diff --git a/integration_tests/common.go b/integration_tests/common.go new file mode 100644 index 00000000..5739b710 --- /dev/null +++ b/integration_tests/common.go @@ -0,0 +1,67 @@ +// Copyright 2022-2023, EthStorage. +// For license information, see https://github.com/ethstorage/es-node/blob/main/LICENSE + +package integration + +import ( + "context" + "fmt" + "io" + "log" + "net/http" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/ethclient" + esLog "github.com/ethstorage/go-ethstorage/ethstorage/log" +) + +const ( + pkName = "ES_NODE_SIGNER_PRIVATE_KEY" + l1Endpoint = "http://65.108.236.27:8545" + clefEndpoint = "http://65.108.236.27:8550" +) + +var ( + contractAddr1GB = common.HexToAddress("0xE31DbfB4d12B67eE60690Ad8a5877Ce8D77842ED") + contractAddrDevnet1 = common.HexToAddress("0x9f9F5Fd89ad648f2C000C954d8d9C87743243eC5") + minerAddr = common.HexToAddress("0x534632D6d7aD1fe5f832951c97FDe73E4eFD9a77") + value = hexutil.EncodeUint64(10000000000000) + lg = esLog.NewLogger(esLog.DefaultCLIConfig()) +) + +func callVerify(calldata []byte) error { + ctx := context.Background() + client, err := ethclient.DialContext(ctx, l1Endpoint) + if err != nil { + log.Fatalf("Failed to connect to the Ethereum client: %v", err) + } + defer client.Close() + msg := ethereum.CallMsg{ + To: &contractAddr1GB, + Data: calldata, + } + bs, err := client.CallContract(ctx, msg, nil) + if err != nil { + return err + } + if bs[len(bs)-1] != 1 { + return fmt.Errorf("false") + } + return nil +} + +func readFile() ([]byte, error) { + txt_77k := "https://www.gutenberg.org/files/7266/7266-0.txt" + resp, err := http.Get(txt_77k) + if err != nil { + return nil, fmt.Errorf("error reading blob txtUrl: %v", err) + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error reading blob txtUrl: %v", err) + } + return data, nil +} diff --git a/ethstorage/prover/kzg_poseidon_prover_test.go b/integration_tests/kzg_poseidon_prover_test.go similarity index 87% rename from ethstorage/prover/kzg_poseidon_prover_test.go rename to integration_tests/kzg_poseidon_prover_test.go index 07e7fbaa..27f9a4fa 100644 --- a/ethstorage/prover/kzg_poseidon_prover_test.go +++ b/integration_tests/kzg_poseidon_prover_test.go @@ -1,13 +1,14 @@ // Copyright 2022-2023, EthStorage. // For license information, see https://github.com/ethstorage/es-node/blob/main/LICENSE -package prover +package integration import ( "testing" "github.com/ethereum/go-ethereum/common" esLog "github.com/ethstorage/go-ethstorage/ethstorage/log" + "github.com/ethstorage/go-ethstorage/ethstorage/prover" ) func TestKZGPoseidonProver_GenerateZKProofs(t *testing.T) { @@ -33,7 +34,7 @@ func TestKZGPoseidonProver_GenerateZKProofs(t *testing.T) { wantErr: false, }, } - prv := NewKZGPoseidonProver("", "blob_poseidon.zkey", esLog.NewLogger(esLog.DefaultCLIConfig())) + prv := prover.NewKZGPoseidonProver("", "blob_poseidon.zkey", esLog.NewLogger(esLog.DefaultCLIConfig())) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { proofs, masks, err := prv.GenerateZKProofs(tt.args.encodingKeys, tt.args.chunkIdxes) diff --git a/ethstorage/prover/kzg_prover_test.go b/integration_tests/kzg_prover_test.go similarity index 67% rename from ethstorage/prover/kzg_prover_test.go rename to integration_tests/kzg_prover_test.go index 1c7aa7b4..4372b998 100644 --- a/ethstorage/prover/kzg_prover_test.go +++ b/integration_tests/kzg_prover_test.go @@ -1,35 +1,24 @@ // Copyright 2022-2023, EthStorage. // For license information, see https://github.com/ethstorage/es-node/blob/main/LICENSE -package prover +package integration import ( "bytes" "context" - "fmt" - "io" "log" "math/big" - "net/http" + "os" "testing" + "time" - "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethstorage/go-ethstorage/cmd/es-utils/utils" - esLog "github.com/ethstorage/go-ethstorage/ethstorage/log" -) - -var ( - rpc = "http://65.108.236.27:8545" - chainID = "7011893061" - prvk = "95eb6ffd2ae0b115db4d1f0d58388216f9d026896696a5211d77b5f14eb5badf" - contractAddr = common.HexToAddress("0xE31DbfB4d12B67eE60690Ad8a5877Ce8D77842ED") - value = hexutil.EncodeUint64(10000000000000) + "github.com/ethstorage/go-ethstorage/ethstorage/prover" ) func TestKZGProver_GenerateKZGProof(t *testing.T) { @@ -37,7 +26,7 @@ func TestKZGProver_GenerateKZGProof(t *testing.T) { if err != nil { t.Fatalf("read raw data error = %v", err) } - dataHash := uploadBlob(dataRaw) + dataHash := uploadBlob(t, dataRaw) blobs := utils.EncodeBlobs(dataRaw) blob := blobs[0][:] tests := []struct { @@ -54,7 +43,7 @@ func TestKZGProver_GenerateKZGProof(t *testing.T) { 3293, }, } - p := NewKZGProver(esLog.NewLogger(esLog.DefaultCLIConfig())) + p := prover.NewKZGProver(lg) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { peInput, err := p.GenerateKZGProof(blob, tt.chunkIdx) @@ -75,16 +64,31 @@ func TestKZGProver_GenerateKZGProof(t *testing.T) { } } -func uploadBlob(data []byte) common.Hash { +func uploadBlob(t *testing.T, data []byte) common.Hash { + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + client, err := ethclient.DialContext(ctx, l1Endpoint) + if err != nil { + lg.Crit("Failed to connect to the Ethereum client: %v", err) + } + defer client.Close() + + chainID, err := client.ChainID(ctx) + if err != nil { + t.Fatalf("Get chain id failed %v", err) + } + // key: "0x0000000000000000000000000000000000000000000000000000000000000001" // blobIdx: 0 // length: 128*1024 calldata := "0x4581a920000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000" tx := utils.SendBlobTx( - rpc, - contractAddr, - prvk, + l1Endpoint, + contractAddr1GB, + os.Getenv(pkName), data, true, -1, @@ -93,15 +97,10 @@ func uploadBlob(data []byte) common.Hash { "", "", "10000000000", - chainID, + chainID.String(), calldata, ) log.Printf("Blob transaction submitted %v", tx.Hash()) - ctx := context.Background() - client, err := ethclient.DialContext(ctx, rpc) - if err != nil { - log.Fatal("Failed to connect to the Ethereum client", err) - } receipt, err := bind.WaitMined(ctx, client, tx) if err != nil { log.Fatal("Get transaction receipt err:", err) @@ -120,7 +119,7 @@ func uploadBlob(data []byte) common.Hash { func verifyInclusive(trunkIdx uint64, peInput []byte) error { ctx := context.Background() - client, err := ethclient.DialContext(ctx, rpc) + client, err := ethclient.DialContext(ctx, l1Endpoint) if err != nil { log.Fatalf("Failed to connect to the Ethereum client: %v", err) } @@ -150,38 +149,3 @@ func verifyInclusive(trunkIdx uint64, peInput []byte) error { calldata := append(mid, dataField...) return callVerify(calldata) } - -func callVerify(calldata []byte) error { - ctx := context.Background() - client, err := ethclient.DialContext(ctx, rpc) - if err != nil { - log.Fatalf("Failed to connect to the Ethereum client: %v", err) - } - defer client.Close() - msg := ethereum.CallMsg{ - To: &contractAddr, - Data: calldata, - } - bs, err := client.CallContract(ctx, msg, nil) - if err != nil { - return err - } - if bs[len(bs)-1] != 1 { - return fmt.Errorf("false") - } - return nil -} - -func readFile() ([]byte, error) { - txt_77k := "https://www.gutenberg.org/files/7266/7266-0.txt" - resp, err := http.Get(txt_77k) - if err != nil { - return nil, fmt.Errorf("error reading blob txtUrl: %v", err) - } - defer resp.Body.Close() - data, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("error reading blob txtUrl: %v", err) - } - return data, nil -} diff --git a/ethstorage/node/node_mine_test.go b/integration_tests/node_mine_test.go similarity index 60% rename from ethstorage/node/node_mine_test.go rename to integration_tests/node_mine_test.go index 0d0d9ec3..f11b0051 100644 --- a/ethstorage/node/node_mine_test.go +++ b/integration_tests/node_mine_test.go @@ -1,7 +1,7 @@ // Copyright 2022-2023, EthStorage. // For license information, see https://github.com/ethstorage/es-node/blob/main/LICENSE -package node +package integration import ( "context" @@ -18,20 +18,20 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethstorage/go-ethstorage/cmd/es-utils/utils" + "github.com/ethstorage/go-ethstorage/ethstorage" es "github.com/ethstorage/go-ethstorage/ethstorage" "github.com/ethstorage/go-ethstorage/ethstorage/eth" - esLog "github.com/ethstorage/go-ethstorage/ethstorage/log" "github.com/ethstorage/go-ethstorage/ethstorage/miner" "github.com/ethstorage/go-ethstorage/ethstorage/p2p/protocol" + "github.com/ethstorage/go-ethstorage/ethstorage/prover" "github.com/ethstorage/go-ethstorage/ethstorage/signer" "github.com/ethstorage/go-ethstorage/ethstorage/storage" - "golang.org/x/term" ) const ( - rpcUrl = "http://65.108.236.27:8545" kvEntriesPerShard = 8192 kvSize = 128 * 1024 chunkSize = 128 * 1024 @@ -39,31 +39,21 @@ const ( dataFileName = "shard-%d.dat" ) -var ( - minerAddr = common.HexToAddress("0x04580493117292ba13361D8e9e28609ec112264D") - contractAddr = common.HexToAddress("0xE31DbfB4d12B67eE60690Ad8a5877Ce8D77842ED") - private = "95eb6ffd2ae0b115db4d1f0d58388216f9d026896696a5211d77b5f14eb5badf" - shardIds = []uint64{0, 1} - l = esLog.NewLogger(esLog.CLIConfig{ - Level: "info", - Format: "text", - Color: term.IsTerminal(int(os.Stdout.Fd())), - }) -) +var shardIds = []uint64{0, 1} // to use a new contract and override data files: // go test -run ^TestMining$ github.com/ethstorage/go-ethstorage/ethstorage/node -v count=1 -contract=0x9BE8dEbb712A3c0dD163Da85D3aF867793Aef3E6 -init=true func TestMining(t *testing.T) { - contract := contractAddr + contract := contractAddr1GB flagContract := getArg("-contract") if flagContract != "" { contract = common.HexToAddress(flagContract) - l.Info("Use the contract address from flag", "contract", flagContract) + lg.Info("Use the contract address from flag", "contract", flagContract) } init := false flagInitStorage := getArg("-init") if flagInitStorage == "true" { - l.Info("Data files will be removed") + lg.Info("Data files will be removed") init = true } @@ -82,13 +72,13 @@ func TestMining(t *testing.T) { if existFile == 2 { intialized = true - l.Info("Data files already exist, will start mining") + lg.Info("Data files already exist, will start mining") } if existFile == 1 { - l.Crit("One of the data files is missing, please check") + lg.Crit("One of the data files is missing, please check") } if existFile == 0 { - l.Info("Will initialize the data files, please make sure you use a new contract") + lg.Info("Will initialize the data files, please make sure you use a new contract") } storConfig := storage.StorageConfig{ KvSize: kvSize, @@ -113,9 +103,7 @@ func TestMining(t *testing.T) { storConfig.Filenames = files signerCfg := signer.CLIConfig{ - // Endpoint: "http://65.108.236.27:8550", - // Address: "0x13259366de990b0431e2c97cea949362bb68df12", - PrivateKey: private, + PrivateKey: os.Getenv(pkName), } factory, addrFrom, err := signer.SignerFactoryFromConfig(signerCfg) if err != nil { @@ -129,32 +117,61 @@ func TestMining(t *testing.T) { t.Fatalf("Get zkWorkingDir error: %v", err) } miningConfig.ZKWorkingDir = zkWorkingDir - l1 := ð.L1EndpointConfig{ - L1NodeAddr: rpcUrl, - } - cfg := &Config{ - Storage: storConfig, - L1: *l1, - L1EpochPollInterval: time.Second * 10, - Mining: miningConfig, + + pClient, err := eth.Dial(l1Endpoint, contractAddr1GB, lg) + if err != nil { + t.Fatalf("Failed to connect to the Ethereum client: %v", err) } - n, err := New(context.Background(), cfg, l, "") + shardManager, err := initShardManager(storConfig) if err != nil { - t.Fatalf("Create new node failed %v", err) + t.Fatalf("init shard manager error: %v", err) } + storageManager := ethstorage.NewStorageManager(shardManager, pClient) + resourcesCtx, close := context.WithCancel(context.Background()) + feed := new(event.Feed) + + l1api := miner.NewL1MiningAPI(pClient, lg) + pvr := prover.NewKZGPoseidonProver(zkWorkingDir, "blob_poseidon.zkey", lg) + mnr := miner.New(miningConfig, storageManager, l1api, &pvr, feed, lg) + log.Info("Initialized miner") + + l1HeadsSub := event.ResubscribeErr(time.Second*10, func(ctx context.Context, err error) (event.Subscription, error) { + if err != nil { + lg.Warn("Resubscribing after failed L1 subscription", "err", err) + } + return eth.WatchHeadChanges(resourcesCtx, pClient, func(ctx context.Context, sig eth.L1BlockRef) { + select { + case mnr.ChainHeadCh <- sig: + default: + // Channel is full, skipping + } + }) + }) + // l1 := ð.L1EndpointConfig{ + // L1NodeAddr: l1Endpoint, + // } + // cfg := &node.Config{ + // Storage: storConfig, + // L1: *l1, + // L1EpochPollInterval: time.Second * 10, + // Mining: miningConfig, + // } + // n, err := node.New(context.Background(), cfg, lg, "") + // if err != nil { + // t.Fatalf("Create new node failed %v", err) + // } if !intialized { // prepareData(t, n, contract) - fillEmpty(t, n, contract) + fillEmpty(t, pClient, storageManager) } - n.startL1(cfg) - n.miner.Start() - n.feed.Send(protocol.EthStorageSyncDone{ + mnr.Start() + feed.Send(protocol.EthStorageSyncDone{ DoneType: protocol.SingleShardDone, ShardId: 0, }) time.Sleep(360 * time.Second) - n.feed.Send(protocol.EthStorageSyncDone{ + feed.Send(protocol.EthStorageSyncDone{ DoneType: protocol.SingleShardDone, ShardId: 1, }) @@ -162,42 +179,55 @@ func TestMining(t *testing.T) { c := make(chan os.Signal) signal.Notify(c, os.Interrupt, syscall.SIGTERM) <-c - n.miner.Close() + mnr.Close() + + l1HeadsSub.Unsubscribe() + close() } -func getArg(paramName string) string { - for _, arg := range os.Args { - pair := strings.Split(arg, "=") - if len(pair) == 2 && pair[0] == paramName { - paramValue := pair[1] - fmt.Printf("%s=%s\n", paramName, paramValue) - return paramValue +func initShardManager(storConfig storage.StorageConfig) (*es.ShardManager, error) { + shardManager := ethstorage.NewShardManager(storConfig.L1Contract, storConfig.KvSize, storConfig.KvEntriesPerShard, storConfig.ChunkSize) + for _, filename := range storConfig.Filenames { + var err error + var df *ethstorage.DataFile + df, err = ethstorage.OpenDataFile(filename) + if err != nil { + return nil, fmt.Errorf("open failed: %w", err) + } + if df.Miner() != storConfig.Miner { + lg.Error("Miners mismatch", "fromDataFile", df.Miner(), "fromConfig", storConfig.Miner) + return nil, fmt.Errorf("miner mismatches datafile") } + shardManager.AddDataFileAndShard(df) } - return "" + + if shardManager.IsComplete() != nil { + return nil, fmt.Errorf("shard is not completed") + } + return shardManager, nil } -func fillEmpty(t *testing.T, n *EsNode, contract common.Address) { +func fillEmpty(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage.StorageManager) { empty := make([]byte, 0) - block, err := n.l1Source.BlockNumber(context.Background()) + block, err := l1Client.BlockNumber(context.Background()) if err != nil { t.Fatalf("Failed to get block number %v", err) } - n.storageManager.Reset(int64(block)) - lastBlobIdx, err := n.storageManager.LastKvIndex() + storageMgr.Reset(int64(block)) + lastBlobIdx, err := storageMgr.LastKvIndex() if err != nil { t.Fatalf("get lastBlobIdx for FillEmptyKV fail, err: %s", err.Error()) } - limit := n.storageManager.KvEntries() * uint64(len(shardIds)) + limit := storageMgr.KvEntries() * uint64(len(shardIds)) for idx := lastBlobIdx; idx < limit; idx++ { - err = n.storageManager.CommitBlob(idx, empty, common.Hash{}) + err = storageMgr.CommitBlob(idx, empty, common.Hash{}) if err != nil { t.Fatalf("write empty to kv file fail, index: %d; error: %s", idx, err.Error()) } } } -func prepareData(t *testing.T, n *EsNode, contract common.Address) { +func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage.StorageManager) { data, err := getSourceData() if err != nil { t.Fatalf("Get source data failed %v", err) @@ -216,7 +246,7 @@ func prepareData(t *testing.T, n *EsNode, contract common.Address) { t.Logf("tx len %d \n", txs) ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() - chainID, err := n.l1Source.ChainID(ctx) + chainID, err := l1Client.ChainID(ctx) if err != nil { t.Fatalf("Get chain id failed %v", err) } @@ -233,7 +263,7 @@ func prepareData(t *testing.T, n *EsNode, contract common.Address) { if len(blobData) == 0 { break } - kvIdxes, dataHashes, err := utils.UploadBlobs(n.l1Source, rpcUrl, private, chainID.String(), contract, blobData, false, value) + kvIdxes, dataHashes, err := utils.UploadBlobs(l1Client, l1Endpoint, os.Getenv(pkName), chainID.String(), storageMgr.ContractAddress(), blobData, false, value) if err != nil { t.Fatalf("Upload blobs failed %v", err) } @@ -242,13 +272,13 @@ func prepareData(t *testing.T, n *EsNode, contract common.Address) { hashs = append(hashs, dataHashes...) ids = append(ids, kvIdxes...) } - block, err := n.l1Source.BlockNumber(ctx) + block, err := l1Client.BlockNumber(ctx) if err != nil { t.Fatalf("Failed to get block number %v", err) } - n.storageManager.Reset(int64(block)) + storageMgr.Reset(int64(block)) for i := 0; i < len(shardIds)*kvEntriesPerShard; i++ { - err := n.storageManager.CommitBlob(ids[i], blobs[i][:], hashs[i]) + err := storageMgr.CommitBlob(ids[i], blobs[i][:], hashs[i]) if err != nil { t.Fatalf("Failed to commit blob: id %d, error: %v", ids[i], err) } @@ -260,13 +290,13 @@ func CreateDataFiles(cfg *storage.StorageConfig) ([]string, error) { for _, shardIdx := range shardIds { fileName := fmt.Sprintf(dataFileName, shardIdx) if _, err := os.Stat(fileName); err == nil { - l.Crit("Creating data file", "error", "file already exists, will not overwrite", "file", fileName) + lg.Crit("Creating data file", "error", "file already exists, will not overwrite", "file", fileName) } if cfg.ChunkSize == 0 { - l.Crit("Creating data file", "error", "chunk size should not be 0") + lg.Crit("Creating data file", "error", "chunk size should not be 0") } if cfg.KvSize%cfg.ChunkSize != 0 { - l.Crit("Creating data file", "error", "max kv size %% chunk size should be 0") + lg.Crit("Creating data file", "error", "max kv size %% chunk size should be 0") } chunkPerKv := cfg.KvSize / cfg.ChunkSize startChunkId := shardIdx * cfg.KvEntriesPerShard * chunkPerKv @@ -278,7 +308,7 @@ func CreateDataFiles(cfg *storage.StorageConfig) ([]string, error) { log.Crit("Creating data file", "error", err) } - l.Info("Data file created", "shard", shardIdx, "file", fileName, "datafile", fmt.Sprintf("%+v", df)) + lg.Info("Data file created", "shard", shardIdx, "file", fileName, "datafile", fmt.Sprintf("%+v", df)) files = append(files, fileName) } return files, nil @@ -297,3 +327,15 @@ func getSourceData() ([]byte, error) { } return data, nil } + +func getArg(paramName string) string { + for _, arg := range os.Args { + pair := strings.Split(arg, "=") + if len(pair) == 2 && pair[0] == paramName { + paramValue := pair[1] + fmt.Printf("%s=%s\n", paramName, paramValue) + return paramValue + } + } + return "" +} diff --git a/ethstorage/signer/signature_test.go b/integration_tests/signature_test.go similarity index 82% rename from ethstorage/signer/signature_test.go rename to integration_tests/signature_test.go index 6e47c7f1..5703b3a3 100644 --- a/ethstorage/signer/signature_test.go +++ b/integration_tests/signature_test.go @@ -1,4 +1,7 @@ -package signer +// Copyright 2022-2023, EthStorage. +// For license information, see https://github.com/ethstorage/es-node/blob/main/LICENSE + +package integration import ( "context" @@ -10,70 +13,69 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/params" + "github.com/ethstorage/go-ethstorage/ethstorage/signer" ) -const l1Endpoint = "http://65.108.236.27:8545" - func TestSignerFactoryFromConfig(t *testing.T) { tests := []struct { name string - signerConfig CLIConfig + signerConfig signer.CLIConfig addrFrom common.Address addrTo common.Address wantErr bool }{ { "", - CLIConfig{ + signer.CLIConfig{ PrivateKey: "0xaaa279031ebf27a046278a8ff5d1b8ab77362d19ace646653b76a7c1184516ef", Mnemonic: "", HDPath: "", }, common.HexToAddress("0x9ce0b38e90cd2a0c82409486078af83c790dff20"), - common.HexToAddress("0x188aac000e21ec314C5694bB82035b72210315A8"), + contractAddrDevnet1, false, }, { "", - CLIConfig{ + signer.CLIConfig{ PrivateKey: "", Mnemonic: "candy maple cake sugar pudding cream honey rich smooth crumble sweet treat", HDPath: "m/44'/60'/0'/0/0", }, common.HexToAddress("0x627306090abab3a6e1400e9345bc60c78a8bef57"), - common.HexToAddress("0x188aac000e21ec314C5694bB82035b72210315A8"), + contractAddrDevnet1, false, }, { "", - CLIConfig{ + signer.CLIConfig{ PrivateKey: "", Mnemonic: "", HDPath: "", - Endpoint: "http://65.108.236.27:8550", + Endpoint: clefEndpoint, Address: "0x13259366de990b0431e2c97cea949362bb68df12", }, common.HexToAddress("0x13259366DE990B0431E2C97CEa949362BB68df12"), - common.HexToAddress("0x188aac000e21ec314C5694bB82035b72210315A8"), + contractAddrDevnet1, false, }, { "", - CLIConfig{ + signer.CLIConfig{ PrivateKey: "", Mnemonic: "", HDPath: "", - Endpoint: "http://65.108.236.27:8550", + Endpoint: clefEndpoint, Address: "0x13259366de990b0431e2c97cea949362bb68df12", }, common.HexToAddress("0x13259366de990b0431e2c97cea949362bb68df12"), - common.HexToAddress("0x0000000000000000000000000000000000001234"), // only allowed to send tx to 0x188aac000e21ec314C5694bB82035b72210315A8 + common.HexToAddress("0x0000000000000000000000000000000000001234"), // only allowed to send tx to contractAddrDevnet1 true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - factory, addrFrom, err := SignerFactoryFromConfig(tt.signerConfig) + factory, addrFrom, err := signer.SignerFactoryFromConfig(tt.signerConfig) if err != nil { t.Fatal("SignerFactoryFromConfig err", err) } diff --git a/ethstorage/prover/zk_prover_test.go b/integration_tests/zk_prover_test.go similarity index 81% rename from ethstorage/prover/zk_prover_test.go rename to integration_tests/zk_prover_test.go index f5cb3806..fb8211aa 100644 --- a/ethstorage/prover/zk_prover_test.go +++ b/integration_tests/zk_prover_test.go @@ -1,7 +1,7 @@ // Copyright 2022-2023, EthStorage. // For license information, see https://github.com/ethstorage/es-node/blob/main/LICENSE -package prover +package integration import ( "context" @@ -15,9 +15,8 @@ import ( "reflect" "strings" "testing" - "time" - esLog "github.com/ethstorage/go-ethstorage/ethstorage/log" + "github.com/ethstorage/go-ethstorage/ethstorage/prover" "github.com/consensys/gnark-crypto/ecc/bn254/fr" "github.com/crate-crypto/go-proto-danksharding-crypto/eth" @@ -28,6 +27,13 @@ import ( "github.com/ethstorage/go-ethstorage/ethstorage/encoder" ) +const ( + snarkLibDir = "snarkjs" + snarkBuildDir = "snarkbuild" + proofName = "proof_blob_poseidon.json" + publicName = "public_blob_poseidon.json" +) + func TestZKProver_GenerateZKProof(t *testing.T) { type args struct { encodingKey common.Hash @@ -58,7 +64,9 @@ func TestZKProver_GenerateZKProof(t *testing.T) { false, }, } - lg := esLog.NewLogger(esLog.DefaultCLIConfig()) + path, _ := filepath.Abs("../ethstorage/prover") + libDir := filepath.Join(path, snarkLibDir) + p := prover.NewZKProverInternal(path, "blob_poseidon.zkey", lg) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { maskGo, err := GenerateMask(tt.args.encodingKey, tt.args.chunkIdx) @@ -66,13 +74,12 @@ func TestZKProver_GenerateZKProof(t *testing.T) { t.Errorf("GenerateMask() error = %v, wantErr %v", err, tt.wantErr) return } - p := newZKProver("", "blob_poseidon.zkey", false, lg) proof, mask, err := p.GenerateZKProof(tt.args.encodingKey, tt.args.chunkIdx) if (err != nil) != tt.wantErr { t.Errorf("ZKProver.GenerateZKProof() error = %v, wantErr %v", err, tt.wantErr) return } - buildDir := filepath.Join(snarkBuildDir, strings.Join([]string{ + buildDir := filepath.Join(path, snarkBuildDir, strings.Join([]string{ tt.args.encodingKey.Hex(), fmt.Sprint(tt.args.chunkIdx), }, "-")) @@ -88,26 +95,21 @@ func TestZKProver_GenerateZKProof(t *testing.T) { t.Errorf("ZKProver.GenerateZKProof() mask = %v, GenerateMask %v", mask, maskBig) return } - err = localVerify(buildDir) + err = localVerify(t, libDir, buildDir) if (err != nil) != tt.wantErr { t.Errorf("ZKProver.GenerateZKProof() localVerify failed: %v", err) } err = verifyDecodeSample(proof, tt.args.chunkIdx, tt.args.encodingKey, mask) if (err != nil) != tt.wantErr { - t.Errorf("ZKProver.GenerateZKProof() decodeSample err: %v", err) + t.Errorf("ZKProver.GenerateZKProof() verifyDecodeSample err: %v", err) } + t.Log("verifyDecodeSample success!") os.RemoveAll(buildDir) }) } } func GenerateMask(encodingKey common.Hash, chunkIdx uint64) (common.Hash, error) { - start := time.Now() - defer func(start time.Time) { - dur := time.Since(start) - log.Printf("GenerateMask() took %0.0f seconds to execute", dur.Seconds()) - }(start) - if int(chunkIdx) >= eth.FieldElementsPerBlob { return common.Hash{}, fmt.Errorf("chunk index out of scope") } @@ -118,7 +120,6 @@ func GenerateMask(encodingKey common.Hash, chunkIdx uint64) (common.Hash, error) } bytesIdx := chunkIdx * 32 mask := masks[bytesIdx : bytesIdx+32] - log.Printf("generate mask done %x", mask) return common.BytesToHash(mask), nil } @@ -128,7 +129,7 @@ func readXIn(buildDir string) (string, error) { return "", err } defer f.Close() - var input InputPair + var input prover.InputPair var decoder *json.Decoder = json.NewDecoder(f) err = decoder.Decode(&input) if err != nil { @@ -137,23 +138,24 @@ func readXIn(buildDir string) (string, error) { return input.XIn, nil } -func localVerify(buildDir string) error { +func localVerify(t *testing.T, libDir string, buildDir string) error { cmd := exec.Command("snarkjs", "groth16", "verify", - filepath.Join(snarkLibDir, "blob_poseidon_verification_key.json"), + filepath.Join(libDir, "blob_poseidon_verification_key.json"), filepath.Join(buildDir, publicName), filepath.Join(buildDir, proofName), ) out, err := cmd.Output() if err != nil { - log.Printf("Local verify failed %v", err) + t.Logf("Local verify failed: %v, cmd: %s, output: %v", err, cmd.String(), string(out)) + return err } - log.Printf("Local verify done. result: %s", out) + t.Logf("Local verify done. result: %s", out) return nil } -func verifyDecodeSample(proof ZKProof, trunkIdx uint64, encodingKey, mask common.Hash) error { +func verifyDecodeSample(proof prover.ZKProof, trunkIdx uint64, encodingKey, mask common.Hash) error { ctx := context.Background() - client, err := ethclient.DialContext(ctx, rpc) + client, err := ethclient.DialContext(ctx, l1Endpoint) if err != nil { log.Fatalf("Failed to connect to the Ethereum client: %v", err) } @@ -164,7 +166,6 @@ func verifyDecodeSample(proof ZKProof, trunkIdx uint64, encodingKey, mask common maskBN := new(big.Int).SetBytes(mask[:]) h := crypto.Keccak256Hash([]byte("decodeSample(((uint256,uint256),(uint256[2],uint256[2]),(uint256,uint256)),uint256,uint256,uint256)")) - mid := h[0:4] uintType, _ := abi.NewType("uint256", "", nil) proofType, _ := abi.NewType("tuple", "", []abi.ArgumentMarshaling{ { @@ -197,6 +198,6 @@ func verifyDecodeSample(proof ZKProof, trunkIdx uint64, encodingKey, mask common if err != nil { return fmt.Errorf("Err: %v, args.Pack: %v", err, values) } - calldata := append(mid, dataField...) + calldata := append(h[0:4], dataField...) return callVerify(calldata) } From f38517b889548377443e10d5094ebb560873506a Mon Sep 17 00:00:00 2001 From: syntrust Date: Fri, 10 Nov 2023 19:23:30 +0800 Subject: [PATCH 05/14] fix clef and invalid addr --- integration_tests/signature_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/signature_test.go b/integration_tests/signature_test.go index 5703b3a3..51240470 100644 --- a/integration_tests/signature_test.go +++ b/integration_tests/signature_test.go @@ -27,11 +27,11 @@ func TestSignerFactoryFromConfig(t *testing.T) { { "", signer.CLIConfig{ - PrivateKey: "0xaaa279031ebf27a046278a8ff5d1b8ab77362d19ace646653b76a7c1184516ef", + PrivateKey: "0x7fb8f46cff75dd565c23e83f3c5aa2693f39cb6a5ede0120666af139cded39af", Mnemonic: "", HDPath: "", }, - common.HexToAddress("0x9ce0b38e90cd2a0c82409486078af83c790dff20"), + common.HexToAddress("0xd7cc258C5438a392cA1D7873020d5B9971568c00"), contractAddrDevnet1, false, }, From cf1c770df0f027ed472a701572ff81d00433b6f5 Mon Sep 17 00:00:00 2001 From: syntrust Date: Mon, 20 Nov 2023 12:29:25 +0800 Subject: [PATCH 06/14] merge --- .github/workflows/github-actions-es.yml | 23 +++ Dockerfile | 5 + GUIDE.md | 4 +- README.md | 2 +- cmd/es-node/main.go | 7 + cmd/es-utils/utils/utils_test.go | 1 + ethstorage/miner/miner_test.go | 2 + ethstorage/node/node_test.go | 2 + ethstorage/p2p/protocol/sync_test.go | 2 + ethstorage/storage_manager_test.go | 163 +++++++++--------- integration_tests/kzg_poseidon_prover_test.go | 2 + integration_tests/kzg_prover_test.go | 2 + integration_tests/node_mine_test.go | 1 + integration_tests/signature_test.go | 2 + integration_tests/zk_prover_test.go | 2 + run-docker.sh | 41 ++++- run.sh | 22 ++- 17 files changed, 189 insertions(+), 94 deletions(-) create mode 100644 .github/workflows/github-actions-es.yml diff --git a/.github/workflows/github-actions-es.yml b/.github/workflows/github-actions-es.yml new file mode 100644 index 00000000..16d105b7 --- /dev/null +++ b/.github/workflows/github-actions-es.yml @@ -0,0 +1,23 @@ +# This workflow will build a golang project +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go +name: GitHub Actions ES +run-name: ${{ github.actor }} is push code to main 🚀 +on: + push: + branches: [ "main" ] +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.20' + + - name: Build + run: go build -v ./cmd/... + + - name: Test + run: go test -v ./... -tags ci diff --git a/Dockerfile b/Dockerfile index ef4d68f5..fdd00ef0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,11 @@ FROM golang:1.20-alpine as builder RUN apk add --no-cache gcc musl-dev linux-headers +# Get dependencies - will also be cached if we won't change go.mod/go.sum +COPY go.mod /es-node/ +COPY go.sum /es-node/ +RUN cd /es-node && go mod download + ADD . /es-node RUN cd /es-node/cmd/es-node && go build RUN cd /es-node/cmd/es-utils && go build diff --git a/GUIDE.md b/GUIDE.md index 40fb6711..91baec9e 100644 --- a/GUIDE.md +++ b/GUIDE.md @@ -19,7 +19,7 @@ For a detailed explanation for es-node please refer to the [README](/README.md). ## System Environment - Ubuntu 20.04+ (has been tested with) - (Optional) Docker 24.0.5+ (would simplify the process) - - (Optional) go 1.20+ and node 16+ (can be installed following the [steps](#1-install-go-120-eg-v1213)) + - (Optional) go 1.20.* (can't be built on Go 1.21 yet) and node 16+ (can be installed following the [steps](#1-install-go-120-eg-v1213)) You can choose [how to run es-node](#step-3-run-es-node) according to your current environment. ## Step 1. Prepare miner and signer account @@ -32,7 +32,7 @@ Remember to use the signer's private key (with ETH balance) to replace ` ES_NODE_SIGNER_PRIVATE_KEY= ./run-docker.sh +if [ -z "$ES_NODE_STORAGE_MINER" ]; then + echo "Please provide 'ES_NODE_STORAGE_MINER' as an environment variable" + exit 1 +fi + +if [ ${#ES_NODE_STORAGE_MINER} -ne 42 ] || case $ES_NODE_STORAGE_MINER in 0x*) false;; *) true;; esac; then + echo "Error: ES_NODE_STORAGE_MINER should be prefixed with '0x' and have a total length of 42" + exit 1 +fi + +if [ -z "$ES_NODE_SIGNER_PRIVATE_KEY" ]; then + echo "Please provide 'ES_NODE_SIGNER_PRIVATE_KEY' as an environment variable" + exit 1 +fi + +if [ ${#ES_NODE_SIGNER_PRIVATE_KEY} -ne 64 ]; then + echo "Error: ES_NODE_SIGNER_PRIVATE_KEY should have a length of 64" + exit 1 +fi + container_name="es" image_name="es-node" # check if container is running -if docker ps --format "{{.Names}}" | grep -q "^$container_name$"; then +if sudo docker ps --format "{{.Names}}" | grep -q "^$container_name$"; then echo "container $container_name already started" else # start container if exist - if docker ps -a --format "{{.Names}}" | grep -q "^$container_name$"; then - docker start $container_name + if sudo docker ps -a --format "{{.Names}}" | grep -q "^$container_name$"; then + sudo docker start $container_name echo "container $container_name started" else # build an image if not exist - if ! docker images --format "{{.Repository}}" | grep -q "^$image_name$"; then - docker build -t $image_name . + if ! sudo docker images --format "{{.Repository}}" | grep -q "^$image_name$"; then + sudo docker build -t $image_name . echo "image $image_name built" fi # run container in the background - docker run --name $container_name -v ./es-data:/es-node/es-data -e ES_NODE_STORAGE_MINER=$ES_NODE_STORAGE_MINER -e ES_NODE_SIGNER_PRIVATE_KEY=$ES_NODE_SIGNER_PRIVATE_KEY -p 9545:9545 -p 9222:9222 -p 30305:30305/udp -d --entrypoint /es-node/run.sh $image_name + sudo docker run --name $container_name \ + -v ./es-data:/es-node/es-data \ + -e ES_NODE_STORAGE_MINER=$ES_NODE_STORAGE_MINER \ + -e ES_NODE_SIGNER_PRIVATE_KEY=$ES_NODE_SIGNER_PRIVATE_KEY \ + -p 9545:9545 \ + -p 9222:9222 \ + -p 30305:30305/udp \ + -d \ + --entrypoint /es-node/run.sh \ + $image_name echo "container $container_name started" fi fi diff --git a/run.sh b/run.sh index 25ff6862..5044b5e1 100644 --- a/run.sh +++ b/run.sh @@ -4,12 +4,22 @@ # env ES_NODE_STORAGE_MINER= ES_NODE_SIGNER_PRIVATE_KEY= ./run.sh if [ -z "$ES_NODE_STORAGE_MINER" ]; then - echo "Please provide 'ES_NODE_STORAGE_MINER' as environment variable" + echo "Please provide 'ES_NODE_STORAGE_MINER' as an environment variable" + exit 1 +fi + +if [ ${#ES_NODE_STORAGE_MINER} -ne 42 ] || case $ES_NODE_STORAGE_MINER in 0x*) false;; *) true;; esac; then + echo "Error: ES_NODE_STORAGE_MINER should be prefixed with '0x' and have a total length of 42" exit 1 fi if [ -z "$ES_NODE_SIGNER_PRIVATE_KEY" ]; then - echo "Please provide 'ES_NODE_SIGNER_PRIVATE_KEY' as environment variable" + echo "Please provide 'ES_NODE_SIGNER_PRIVATE_KEY' as an environment variable" + exit 1 +fi + +if [ ${#ES_NODE_SIGNER_PRIVATE_KEY} -ne 64 ]; then + echo "Error: ES_NODE_SIGNER_PRIVATE_KEY should have a length of 64" exit 1 fi @@ -55,8 +65,12 @@ es_node_start=" --network devnet \ " # create data file for shard 0 if not yet if [ ! -e $storage_file_0 ]; then - $executable $es_node_init $common_flags - echo "initialized ${storage_file_0}" + if $executable $es_node_init $common_flags ; then + echo "initialized ${storage_file_0} successfully" + else + echo "failed to initialize ${storage_file_0}" + exit 1 + fi fi # start es-node From 45d396be5c7eb26277d3c6eea44993c32c8d55f3 Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 21 Nov 2023 19:18:46 +0800 Subject: [PATCH 07/14] improve log --- ethstorage/miner/worker.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ethstorage/miner/worker.go b/ethstorage/miner/worker.go index b9de3d24..6104fa2a 100644 --- a/ethstorage/miner/worker.go +++ b/ethstorage/miner/worker.go @@ -353,10 +353,9 @@ func (w *worker) mineTask(t *taskItem) (bool, error) { startTime := time.Now() nonce := t.nonceStart if t.thread == 0 { - w.lg.Info("Mining task started", "shard", t.shardIdx, "thread", t.thread, "block", t.blockNumber, "nonce", fmt.Sprintf("%d~%d", t.nonceStart, t.nonceEnd)) - } else { - w.lg.Debug("Mining task started", "shard", t.shardIdx, "thread", t.thread, "block", t.blockNumber, "nonce", fmt.Sprintf("%d~%d", t.nonceStart, t.nonceEnd)) + w.lg.Info("Mining tasks started", "shard", t.shardIdx, "threads", w.config.ThreadsPerShard, "block", t.blockNumber, "nonces", fmt.Sprintf("%d~%d", 0, w.config.NonceLimit)) } + w.lg.Debug("Mining task started", "shard", t.shardIdx, "thread", t.thread, "block", t.blockNumber, "nonces", fmt.Sprintf("%d~%d", t.nonceStart, t.nonceEnd)) for w.isRunning() { if time.Since(startTime).Seconds() > mineTimeOut { w.lg.Warn("Mining task timed out", "shard", t.shardIdx, "thread", t.thread, "block", t.blockNumber, "noncesTried", nonce-t.nonceStart) From c781d23055c14d3130431dfe9f5dbf6827271aa0 Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 5 Dec 2023 17:40:20 +0800 Subject: [PATCH 08/14] read params from contract --- cmd/es-utils/utils/utils.go | 12 +++--- integration_tests/common.go | 49 +++++++++++++++++++++- integration_tests/node_mine_test.go | 64 +++++++++++------------------ 3 files changed, 78 insertions(+), 47 deletions(-) diff --git a/cmd/es-utils/utils/utils.go b/cmd/es-utils/utils/utils.go index 83bd003f..63972c2b 100644 --- a/cmd/es-utils/utils/utils.go +++ b/cmd/es-utils/utils/utils.go @@ -57,7 +57,7 @@ func SendBlobTx( h := crypto.Keccak256Hash([]byte(`upfrontPayment()`)) callMsg := ethereum.CallMsg{ - To: &to, + To: &to, Data: h[:], } bs, err := client.CallContract(context.Background(), callMsg, new(big.Int).SetInt64(-2)) @@ -76,8 +76,8 @@ func SendBlobTx( log.Crit("Invalid value param") } - if res[0].(* big.Int).Cmp(val) == 1 { - val = res[0].(* big.Int) + if res[0].(*big.Int).Cmp(val) == 1 { + val = res[0].(*big.Int) } value256, overflow := uint256.FromBig(val) @@ -320,10 +320,10 @@ func UploadBlobs( needEncoding, -1, value, - 1000000, + 5000000, "", - "400000000", - "4000000000", + "", + "30000000000", chainID, calldata, ) diff --git a/integration_tests/common.go b/integration_tests/common.go index 5739b710..cb9b0911 100644 --- a/integration_tests/common.go +++ b/integration_tests/common.go @@ -8,29 +8,74 @@ import ( "fmt" "io" "log" + "math/big" "net/http" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient" esLog "github.com/ethstorage/go-ethstorage/ethstorage/log" + "github.com/ethstorage/go-ethstorage/ethstorage/storage" ) const ( pkName = "ES_NODE_SIGNER_PRIVATE_KEY" - l1Endpoint = "http://65.108.236.27:8545" + l1Endpoint = "http://65.109.115.36:8545" clefEndpoint = "http://65.108.236.27:8550" ) var ( - contractAddr1GB = common.HexToAddress("0xE31DbfB4d12B67eE60690Ad8a5877Ce8D77842ED") + contractAddr1GB = common.HexToAddress("0x0c47aa023A3be36CA148a17a91aA8C9268A350AE") contractAddrDevnet1 = common.HexToAddress("0x9f9F5Fd89ad648f2C000C954d8d9C87743243eC5") minerAddr = common.HexToAddress("0x534632D6d7aD1fe5f832951c97FDe73E4eFD9a77") value = hexutil.EncodeUint64(10000000000000) lg = esLog.NewLogger(esLog.DefaultCLIConfig()) ) +func initStorageConfig(ctx context.Context, client *ethclient.Client, l1Contract, miner common.Address) (*storage.StorageConfig, error) { + maxKvSizeBits, err := readUintFromContract(ctx, client, l1Contract, "maxKvSizeBits") + if err != nil { + return nil, err + } + chunkSizeBits := maxKvSizeBits + shardEntryBits, err := readUintFromContract(ctx, client, l1Contract, "shardEntryBits") + if err != nil { + return nil, err + } + return &storage.StorageConfig{ + L1Contract: l1Contract, + Miner: miner, + KvSize: 1 << maxKvSizeBits, + ChunkSize: 1 << chunkSizeBits, + KvEntriesPerShard: 1 << shardEntryBits, + }, nil +} + +func readSlotFromContract(ctx context.Context, client *ethclient.Client, l1Contract common.Address, fieldName string) ([]byte, error) { + h := crypto.Keccak256Hash([]byte(fieldName + "()")) + msg := ethereum.CallMsg{ + To: &l1Contract, + Data: h[0:4], + } + bs, err := client.CallContract(ctx, msg, nil) + if err != nil { + return nil, fmt.Errorf("Failed to get %s from contract: %v", fieldName, err) + } + return bs, nil +} + +func readUintFromContract(ctx context.Context, client *ethclient.Client, l1Contract common.Address, fieldName string) (uint64, error) { + bs, err := readSlotFromContract(ctx, client, l1Contract, fieldName) + if err != nil { + return 0, err + } + value := new(big.Int).SetBytes(bs).Uint64() + log.Println("Read uint from contract", "field", fieldName, "value", value) + return value, nil +} + func callVerify(calldata []byte) error { ctx := context.Background() client, err := ethclient.DialContext(ctx, l1Endpoint) diff --git a/integration_tests/node_mine_test.go b/integration_tests/node_mine_test.go index 01dbb579..e548f082 100644 --- a/integration_tests/node_mine_test.go +++ b/integration_tests/node_mine_test.go @@ -2,6 +2,7 @@ // For license information, see https://github.com/ethstorage/es-node/blob/main/LICENSE //go:build !ci + package integration import ( @@ -33,19 +34,17 @@ import ( ) const ( - kvEntriesPerShard = 8192 - kvSize = 128 * 1024 - chunkSize = 128 * 1024 - maxBlobsPerTx = 4 - dataFileName = "shard-%d.dat" + maxBlobsPerTx = 4 + dataFileName = "shard-%d.dat" ) -var shardIds = []uint64{0, 1} +var shardIds = []uint64{0} // to use a new contract and override data files: // go test -run ^TestMining$ github.com/ethstorage/go-ethstorage/ethstorage/node -v count=1 -contract=0x9BE8dEbb712A3c0dD163Da85D3aF867793Aef3E6 -init=true func TestMining(t *testing.T) { contract := contractAddr1GB + // contract := common.HexToAddress("0xf8928E47ab82912025FD45589f2444fF3a0FDa73") flagContract := getArg("-contract") if flagContract != "" { contract = common.HexToAddress(flagContract) @@ -75,22 +74,24 @@ func TestMining(t *testing.T) { intialized = true lg.Info("Data files already exist, will start mining") } - if existFile == 1 { - lg.Crit("One of the data files is missing, please check") - } + // if existFile == 1 { + // lg.Crit("One of the data files is missing, please check") + // } if existFile == 0 { lg.Info("Will initialize the data files, please make sure you use a new contract") } - storConfig := storage.StorageConfig{ - KvSize: kvSize, - ChunkSize: chunkSize, - KvEntriesPerShard: kvEntriesPerShard, - L1Contract: contract, - Miner: minerAddr, + + pClient, err := eth.Dial(l1Endpoint, contract, lg) + if err != nil { + t.Fatalf("Failed to connect to the Ethereum client: %v", err) + } + storConfig, err := initStorageConfig(context.Background(), pClient.Client, contract, minerAddr) + if err != nil { + t.Fatalf("Failed to init storage client: %v", err) } var files []string if !intialized { - f, err := CreateDataFiles(&storConfig) + f, err := CreateDataFiles(storConfig) if err != nil { t.Fatalf("Create data files error: %v", err) } @@ -102,7 +103,9 @@ func TestMining(t *testing.T) { } } storConfig.Filenames = files - + if os.Getenv(pkName) == "" { + t.Fatal("No private key in env", pkName) + } signerCfg := signer.CLIConfig{ PrivateKey: os.Getenv(pkName), } @@ -119,11 +122,7 @@ func TestMining(t *testing.T) { } miningConfig.ZKWorkingDir = zkWorkingDir - pClient, err := eth.Dial(l1Endpoint, contractAddr1GB, lg) - if err != nil { - t.Fatalf("Failed to connect to the Ethereum client: %v", err) - } - shardManager, err := initShardManager(storConfig) + shardManager, err := initShardManager(*storConfig) if err != nil { t.Fatalf("init shard manager error: %v", err) } @@ -149,22 +148,9 @@ func TestMining(t *testing.T) { } }) }) - // l1 := ð.L1EndpointConfig{ - // L1NodeAddr: l1Endpoint, - // } - // cfg := &node.Config{ - // Storage: storConfig, - // L1: *l1, - // L1EpochPollInterval: time.Second * 10, - // Mining: miningConfig, - // } - // n, err := node.New(context.Background(), cfg, lg, "") - // if err != nil { - // t.Fatalf("Create new node failed %v", err) - // } if !intialized { - // prepareData(t, n, contract) - fillEmpty(t, pClient, storageManager) + prepareData(t, pClient, storageManager) + // fillEmpty(t, pClient, storageManager) } mnr.Start() feed.Send(protocol.EthStorageSyncDone{ @@ -238,7 +224,7 @@ func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstora var hashs []common.Hash var ids []uint64 - value := hexutil.EncodeUint64(10000000000000) + value := hexutil.EncodeUint64(500000000000000) txs := len(blobs) / maxBlobsPerTx last := len(blobs) % maxBlobsPerTx if last > 0 { @@ -278,7 +264,7 @@ func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstora t.Fatalf("Failed to get block number %v", err) } storageMgr.Reset(int64(block)) - for i := 0; i < len(shardIds)*kvEntriesPerShard; i++ { + for i := 0; i < len(shardIds)*int(storageMgr.KvEntries()); i++ { err := storageMgr.CommitBlob(ids[i], blobs[i][:], hashs[i]) if err != nil { t.Fatalf("Failed to commit blob: id %d, error: %v", ids[i], err) From 32592cebfab36690baa582fb370755f15015d150 Mon Sep 17 00:00:00 2001 From: syntrust Date: Mon, 18 Dec 2023 11:36:24 +0800 Subject: [PATCH 09/14] download zkey --- integration_tests/kzg_poseidon_prover_test.go | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/integration_tests/kzg_poseidon_prover_test.go b/integration_tests/kzg_poseidon_prover_test.go index 59b45f2b..ff3543cc 100644 --- a/integration_tests/kzg_poseidon_prover_test.go +++ b/integration_tests/kzg_poseidon_prover_test.go @@ -6,6 +6,12 @@ package integration import ( + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" "testing" "github.com/ethereum/go-ethereum/common" @@ -13,6 +19,11 @@ import ( "github.com/ethstorage/go-ethstorage/ethstorage/prover" ) +const ( + prPath = "../ethstorage/prover" + zkeyFile = "blob_poseidon.zkey" +) + func TestKZGPoseidonProver_GenerateZKProofs(t *testing.T) { type args struct { encodingKeys []common.Hash @@ -36,7 +47,9 @@ func TestKZGPoseidonProver_GenerateZKProofs(t *testing.T) { wantErr: false, }, } - prv := prover.NewKZGPoseidonProver("", "blob_poseidon.zkey", esLog.NewLogger(esLog.DefaultCLIConfig())) + + proverPath, _ := filepath.Abs(prPath) + prv := prover.NewKZGPoseidonProver(proverPath, zkeyFile, esLog.NewLogger(esLog.DefaultCLIConfig())) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { proofs, masks, err := prv.GenerateZKProofs(tt.args.encodingKeys, tt.args.chunkIdxes) @@ -53,3 +66,38 @@ func TestKZGPoseidonProver_GenerateZKProofs(t *testing.T) { }) } } + +func init() { + proverPath, _ := filepath.Abs(prPath) + zkeyFile := filepath.Join(proverPath, "snarkjs", zkeyFile) + fileID := "1ZLfhYeCXMnbk6wUiBADRAn1mZ8MI_zg-" + if _, err := os.Stat(zkeyFile); os.IsNotExist(err) { + fmt.Printf("%s not found, start downloading...\n", zkeyFile) + cookieCmd := exec.Command("curl", "-c", "./cookie", "-s", "-L", fmt.Sprintf("https://drive.google.com/uc?export=download&id=%s", fileID)) + cookieOutput, err := cookieCmd.Output() + if err != nil { + fmt.Println("Error downloading file:", err) + panic(err) + } + confirmCode := strings.TrimPrefix(strings.TrimSpace(string(cookieOutput)), "confirm=") + downloadCmd := exec.Command("curl", "-Lb", "./cookie", fmt.Sprintf("https://drive.google.com/uc?export=download&confirm=%s&id=%s", confirmCode, fileID)) + downloadOutput, err := downloadCmd.StdoutPipe() + if err != nil { + fmt.Println("Error downloading file:", err) + panic(err) + } + zkeyFileWriter, _ := os.Create(zkeyFile) + defer zkeyFileWriter.Close() + + if err := downloadCmd.Start(); err != nil { + fmt.Println("Error downloading file:", err) + panic(err) + } + + _, _ = io.Copy(zkeyFileWriter, downloadOutput) + + if err := downloadCmd.Wait(); err != nil { + fmt.Println("Error downloading file:", err) + } + } +} From cf23b58276c8ccbd1fc23012d39564d9d918cd84 Mon Sep 17 00:00:00 2001 From: syntrust Date: Mon, 18 Dec 2023 11:36:48 +0800 Subject: [PATCH 10/14] merge --- integration_tests/node_mine_test.go | 11 +---------- integration_tests/zk_prover_test.go | 5 ----- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/integration_tests/node_mine_test.go b/integration_tests/node_mine_test.go index a2c27e6d..f7c67b82 100644 --- a/integration_tests/node_mine_test.go +++ b/integration_tests/node_mine_test.go @@ -200,18 +200,9 @@ func fillEmpty(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage if err != nil { t.Fatalf("Failed to get block number %v", err) } -<<<<<<< HEAD:integration_tests/node_mine_test.go storageMgr.Reset(int64(block)) - lastBlobIdx, err := storageMgr.LastKvIndex() - if err != nil { - t.Fatalf("get lastBlobIdx for FillEmptyKV fail, err: %s", err.Error()) - } limit := storageMgr.KvEntries() * uint64(len(shardIds)) -======= - n.storageManager.Reset(int64(block)) - lastBlobIdx := n.storageManager.LastKvIndex() - limit := n.storageManager.KvEntries() * uint64(len(shardIds)) ->>>>>>> 6985e6968ebf0cb626a7764a9f74907a6870feec:ethstorage/node/node_mine_test.go + lastBlobIdx := storageMgr.LastKvIndex() for idx := lastBlobIdx; idx < limit; idx++ { err = storageMgr.CommitBlob(idx, empty, common.Hash{}) if err != nil { diff --git a/integration_tests/zk_prover_test.go b/integration_tests/zk_prover_test.go index 862731d0..85fd604f 100644 --- a/integration_tests/zk_prover_test.go +++ b/integration_tests/zk_prover_test.go @@ -131,13 +131,8 @@ func readXIn(buildDir string) (string, error) { return "", err } defer f.Close() -<<<<<<< HEAD:integration_tests/zk_prover_test.go var input prover.InputPair var decoder *json.Decoder = json.NewDecoder(f) -======= - var input InputPair - var decoder = json.NewDecoder(f) ->>>>>>> 6985e6968ebf0cb626a7764a9f74907a6870feec:ethstorage/prover/zk_prover_test.go err = decoder.Decode(&input) if err != nil { return "", err From 03f139aac8e6c483e687ee7bf89612c31c3872bf Mon Sep 17 00:00:00 2001 From: syntrust Date: Mon, 18 Dec 2023 12:23:58 +0800 Subject: [PATCH 11/14] leave the zkey download to tester --- integration_tests/kzg_poseidon_prover_test.go | 40 ------------------- 1 file changed, 40 deletions(-) diff --git a/integration_tests/kzg_poseidon_prover_test.go b/integration_tests/kzg_poseidon_prover_test.go index ff3543cc..3c04e5e3 100644 --- a/integration_tests/kzg_poseidon_prover_test.go +++ b/integration_tests/kzg_poseidon_prover_test.go @@ -6,12 +6,7 @@ package integration import ( - "fmt" - "io" - "os" - "os/exec" "path/filepath" - "strings" "testing" "github.com/ethereum/go-ethereum/common" @@ -66,38 +61,3 @@ func TestKZGPoseidonProver_GenerateZKProofs(t *testing.T) { }) } } - -func init() { - proverPath, _ := filepath.Abs(prPath) - zkeyFile := filepath.Join(proverPath, "snarkjs", zkeyFile) - fileID := "1ZLfhYeCXMnbk6wUiBADRAn1mZ8MI_zg-" - if _, err := os.Stat(zkeyFile); os.IsNotExist(err) { - fmt.Printf("%s not found, start downloading...\n", zkeyFile) - cookieCmd := exec.Command("curl", "-c", "./cookie", "-s", "-L", fmt.Sprintf("https://drive.google.com/uc?export=download&id=%s", fileID)) - cookieOutput, err := cookieCmd.Output() - if err != nil { - fmt.Println("Error downloading file:", err) - panic(err) - } - confirmCode := strings.TrimPrefix(strings.TrimSpace(string(cookieOutput)), "confirm=") - downloadCmd := exec.Command("curl", "-Lb", "./cookie", fmt.Sprintf("https://drive.google.com/uc?export=download&confirm=%s&id=%s", confirmCode, fileID)) - downloadOutput, err := downloadCmd.StdoutPipe() - if err != nil { - fmt.Println("Error downloading file:", err) - panic(err) - } - zkeyFileWriter, _ := os.Create(zkeyFile) - defer zkeyFileWriter.Close() - - if err := downloadCmd.Start(); err != nil { - fmt.Println("Error downloading file:", err) - panic(err) - } - - _, _ = io.Copy(zkeyFileWriter, downloadOutput) - - if err := downloadCmd.Wait(); err != nil { - fmt.Println("Error downloading file:", err) - } - } -} From f432473fb10b0b8d642cf50445711be6acb306cf Mon Sep 17 00:00:00 2001 From: syntrust Date: Mon, 18 Dec 2023 18:04:13 +0800 Subject: [PATCH 12/14] fix kzg --- cmd/es-utils/utils/utils.go | 2 +- integration_tests/common.go | 15 ++-- integration_tests/kzg_poseidon_prover_test.go | 10 +-- integration_tests/kzg_prover_test.go | 77 ++++++++++++++----- integration_tests/run_tests.sh | 27 +++++++ integration_tests/signature_test.go | 6 +- integration_tests/zk_prover_test.go | 14 +++- 7 files changed, 109 insertions(+), 42 deletions(-) create mode 100755 integration_tests/run_tests.sh diff --git a/cmd/es-utils/utils/utils.go b/cmd/es-utils/utils/utils.go index 63972c2b..59a67184 100644 --- a/cmd/es-utils/utils/utils.go +++ b/cmd/es-utils/utils/utils.go @@ -323,7 +323,7 @@ func UploadBlobs( 5000000, "", "", - "30000000000", + "300000000", chainID, calldata, ) diff --git a/integration_tests/common.go b/integration_tests/common.go index cb9b0911..1569fb52 100644 --- a/integration_tests/common.go +++ b/integration_tests/common.go @@ -7,13 +7,11 @@ import ( "context" "fmt" "io" - "log" "math/big" "net/http" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient" esLog "github.com/ethstorage/go-ethstorage/ethstorage/log" @@ -27,10 +25,10 @@ const ( ) var ( - contractAddr1GB = common.HexToAddress("0x0c47aa023A3be36CA148a17a91aA8C9268A350AE") - contractAddrDevnet1 = common.HexToAddress("0x9f9F5Fd89ad648f2C000C954d8d9C87743243eC5") + contractAddr1GB = common.HexToAddress("0x24d29453B4B4917117B4af022Ad1B0a9aBc2c7DE") + contractAddrDevnet2 = common.HexToAddress("0xb4B46bdAA835F8E4b4d8e208B6559cD267851051") minerAddr = common.HexToAddress("0x534632D6d7aD1fe5f832951c97FDe73E4eFD9a77") - value = hexutil.EncodeUint64(10000000000000) + value = "1000000000000000" lg = esLog.NewLogger(esLog.DefaultCLIConfig()) ) @@ -61,7 +59,7 @@ func readSlotFromContract(ctx context.Context, client *ethclient.Client, l1Contr } bs, err := client.CallContract(ctx, msg, nil) if err != nil { - return nil, fmt.Errorf("Failed to get %s from contract: %v", fieldName, err) + return nil, fmt.Errorf("failed to get %s from contract: %v", fieldName, err) } return bs, nil } @@ -72,7 +70,7 @@ func readUintFromContract(ctx context.Context, client *ethclient.Client, l1Contr return 0, err } value := new(big.Int).SetBytes(bs).Uint64() - log.Println("Read uint from contract", "field", fieldName, "value", value) + lg.Info("Read uint from contract", "field", fieldName, "value", value) return value, nil } @@ -80,7 +78,8 @@ func callVerify(calldata []byte) error { ctx := context.Background() client, err := ethclient.DialContext(ctx, l1Endpoint) if err != nil { - log.Fatalf("Failed to connect to the Ethereum client: %v", err) + lg.Error("Failed to connect to the Ethereum client", "error", err) + return err } defer client.Close() msg := ethereum.CallMsg{ diff --git a/integration_tests/kzg_poseidon_prover_test.go b/integration_tests/kzg_poseidon_prover_test.go index 3c04e5e3..48ba5c1a 100644 --- a/integration_tests/kzg_poseidon_prover_test.go +++ b/integration_tests/kzg_poseidon_prover_test.go @@ -6,6 +6,7 @@ package integration import ( + "os" "path/filepath" "testing" @@ -14,11 +15,6 @@ import ( "github.com/ethstorage/go-ethstorage/ethstorage/prover" ) -const ( - prPath = "../ethstorage/prover" - zkeyFile = "blob_poseidon.zkey" -) - func TestKZGPoseidonProver_GenerateZKProofs(t *testing.T) { type args struct { encodingKeys []common.Hash @@ -44,6 +40,10 @@ func TestKZGPoseidonProver_GenerateZKProofs(t *testing.T) { } proverPath, _ := filepath.Abs(prPath) + zkeyFull := filepath.Join(proverPath, snarkLibDir, zkeyFile) + if _, err := os.Stat(zkeyFull); os.IsNotExist(err) { + t.Fatalf("%s not found", zkeyFull) + } prv := prover.NewKZGPoseidonProver(proverPath, zkeyFile, esLog.NewLogger(esLog.DefaultCLIConfig())) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/integration_tests/kzg_prover_test.go b/integration_tests/kzg_prover_test.go index f6816a50..595cd1e1 100644 --- a/integration_tests/kzg_prover_test.go +++ b/integration_tests/kzg_prover_test.go @@ -8,17 +8,18 @@ package integration import ( "bytes" "context" - "log" "math/big" "os" "testing" "time" + "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" "github.com/ethstorage/go-ethstorage/cmd/es-utils/utils" "github.com/ethstorage/go-ethstorage/ethstorage/prover" ) @@ -81,41 +82,75 @@ func uploadBlob(t *testing.T, data []byte) common.Hash { if err != nil { t.Fatalf("Get chain id failed %v", err) } + pk := os.Getenv(pkName) + key, err := crypto.HexToECDSA(pk) + if err != nil { + t.Fatalf("Invalid private key: %s, err: %v", pk, err) + } + signer := crypto.PubkeyToAddress(key.PublicKey) + lg.Info("Get signer address", "signer", signer.Hex()) + n, err := client.NonceAt(ctx, signer, big.NewInt(rpc.LatestBlockNumber.Int64())) + if err != nil { + t.Fatalf("Error getting nonce: %v", err) + } + blbKey := "0x0000000000000000000000000000000000000000000000000000000000000001" + blbIdx := common.Big0 + length := new(big.Int).SetInt64(128 * 1024) + // calldata := "0x4581a920000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000" - // key: "0x0000000000000000000000000000000000000000000000000000000000000001" - // blobIdx: 0 - // length: 128*1024 - calldata := "0x4581a920000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000" + h := crypto.Keccak256Hash([]byte("putBlob(bytes32,uint256,uint256)")) + mid := h[0:4] + bytes32Type, _ := abi.NewType("bytes32", "", nil) + uint256Type, _ := abi.NewType("uint256", "", nil) + args := abi.Arguments{ + {Type: bytes32Type}, + {Type: uint256Type}, + {Type: uint256Type}, + } + values := []interface{}{common.HexToHash(blbKey), blbIdx, length} + dataField, err := args.Pack(values...) + if err != nil { + t.Fatalf("Error getting calldata: %v", err) + } + calldata := append(mid, dataField...) + valBig, _ := new(big.Int).SetString(value, 10) + estimatedGas, err := client.EstimateGas(ctx, ethereum.CallMsg{ + From: signer, + To: &contractAddr1GB, + Value: valBig, + Data: calldata, + }) + if err != nil { + lg.Crit("Estimate gas failed", "error", err.Error()) + } + lg.Info("Estimated gas done", "gas", estimatedGas) tx := utils.SendBlobTx( l1Endpoint, contractAddr1GB, - os.Getenv(pkName), + pk, data, true, - -1, + int64(n), value, 510000, "", "", - "10000000000", + "100000000", chainID.String(), - calldata, + "0x"+common.Bytes2Hex(calldata), ) - log.Printf("Blob transaction submitted %v", tx.Hash()) + lg.Info("Blob transaction submitted", "hash", tx.Hash()) receipt, err := bind.WaitMined(ctx, client, tx) if err != nil { - log.Fatal("Get transaction receipt err:", err) - } - if receipt.Status == 0 { - log.Fatal("Blob transaction failed") + lg.Crit("Get transaction receipt err", "error", err) } - log.Printf("Blob transaction success! Gas used %v", receipt.GasUsed) - eventTopics := receipt.Logs[0].Topics - kvIndex := new(big.Int).SetBytes(eventTopics[1][:]) - kvSize := new(big.Int).SetBytes(eventTopics[2][:]) - dataHash := eventTopics[3] - log.Printf("Put Blob with kvIndex=%v kvSize=%v, dataHash=%x", kvIndex, kvSize, dataHash) + // if receipt.Status == 0 { + // lg.Crit("Blob transaction failed") + // } + lg.Info("Blob transaction success!", "GasUsed", receipt.GasUsed) + dataHash := receipt.Logs[0].Topics[3] + lg.Info("Put Blob done", "datahash", dataHash) return dataHash } @@ -123,7 +158,7 @@ func verifyInclusive(trunkIdx uint64, peInput []byte) error { ctx := context.Background() client, err := ethclient.DialContext(ctx, l1Endpoint) if err != nil { - log.Fatalf("Failed to connect to the Ethereum client: %v", err) + lg.Crit("Failed to connect to the Ethereum client.", "error", err) } defer client.Close() diff --git a/integration_tests/run_tests.sh b/integration_tests/run_tests.sh new file mode 100755 index 00000000..b4db3f02 --- /dev/null +++ b/integration_tests/run_tests.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +# usage: +# env ES_NODE_SIGNER_PRIVATE_KEY= ./run_tests.sh + +if [ -z "$ES_NODE_SIGNER_PRIVATE_KEY" ]; then + echo "Please provide 'ES_NODE_SIGNER_PRIVATE_KEY' as an environment variable" + exit 1 +fi + +if [ ${#ES_NODE_SIGNER_PRIVATE_KEY} -ne 64 ]; then + echo "Error: ES_NODE_SIGNER_PRIVATE_KEY should have a length of 64" + exit 1 +fi + +# download blob_poseidon.zkey if not yet +zkey_file="./ethstorage/prover/snarkjs/blob_poseidon.zkey" +if [ ! -e ${zkey_file} ]; then + echo "${zkey_file} not found, start downloading..." + file_id="1ZLfhYeCXMnbk6wUiBADRAn1mZ8MI_zg-" + html=`curl -c ./cookie -s -L "https://drive.google.com/uc?export=download&id=${file_id}"` + curl -Lb ./cookie "https://drive.google.com/uc?export=download&`echo ${html}|grep -Eo 'confirm=[a-zA-Z0-9\-_]+'`&id=${file_id}" -o ${zkey_file} +fi + +# go test -timeout 0 github.com/ethstorage/go-ethstorage/integration_tests + +go test -timeout 0 -run ^TestKZGProver_GenerateKZGProof$ github.com/ethstorage/go-ethstorage/integration_tests diff --git a/integration_tests/signature_test.go b/integration_tests/signature_test.go index d6f22b9a..7da2dcd8 100644 --- a/integration_tests/signature_test.go +++ b/integration_tests/signature_test.go @@ -34,7 +34,7 @@ func TestSignerFactoryFromConfig(t *testing.T) { HDPath: "", }, common.HexToAddress("0xd7cc258C5438a392cA1D7873020d5B9971568c00"), - contractAddrDevnet1, + contractAddrDevnet2, false, }, { @@ -45,7 +45,7 @@ func TestSignerFactoryFromConfig(t *testing.T) { HDPath: "m/44'/60'/0'/0/0", }, common.HexToAddress("0x627306090abab3a6e1400e9345bc60c78a8bef57"), - contractAddrDevnet1, + contractAddrDevnet2, false, }, { @@ -58,7 +58,7 @@ func TestSignerFactoryFromConfig(t *testing.T) { Address: "0x13259366de990b0431e2c97cea949362bb68df12", }, common.HexToAddress("0x13259366DE990B0431E2C97CEa949362BB68df12"), - contractAddrDevnet1, + contractAddrDevnet2, false, }, { diff --git a/integration_tests/zk_prover_test.go b/integration_tests/zk_prover_test.go index 85fd604f..a082caeb 100644 --- a/integration_tests/zk_prover_test.go +++ b/integration_tests/zk_prover_test.go @@ -34,9 +34,16 @@ const ( snarkBuildDir = "snarkbuild" proofName = "proof_blob_poseidon.json" publicName = "public_blob_poseidon.json" + prPath = "../ethstorage/prover" + zkeyFile = "blob_poseidon.zkey" ) func TestZKProver_GenerateZKProof(t *testing.T) { + proverPath, _ := filepath.Abs(prPath) + zkeyFull := filepath.Join(proverPath, snarkLibDir, zkeyFile) + if _, err := os.Stat(zkeyFull); os.IsNotExist(err) { + t.Fatalf("%s not found", zkeyFull) + } type args struct { encodingKey common.Hash chunkIdx uint64 @@ -66,9 +73,8 @@ func TestZKProver_GenerateZKProof(t *testing.T) { false, }, } - path, _ := filepath.Abs("../ethstorage/prover") - libDir := filepath.Join(path, snarkLibDir) - p := prover.NewZKProverInternal(path, "blob_poseidon.zkey", lg) + libDir := filepath.Join(proverPath, snarkLibDir) + p := prover.NewZKProverInternal(proverPath, zkeyFile, lg) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { maskGo, err := GenerateMask(tt.args.encodingKey, tt.args.chunkIdx) @@ -81,7 +87,7 @@ func TestZKProver_GenerateZKProof(t *testing.T) { t.Errorf("ZKProver.GenerateZKProof() error = %v, wantErr %v", err, tt.wantErr) return } - buildDir := filepath.Join(path, snarkBuildDir, strings.Join([]string{ + buildDir := filepath.Join(proverPath, snarkBuildDir, strings.Join([]string{ tt.args.encodingKey.Hex(), fmt.Sprint(tt.args.chunkIdx), }, "-")) From 25e4f1242148c6d98a93277d8b914ff5a0f0d47b Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 19 Dec 2023 16:59:13 +0800 Subject: [PATCH 13/14] fix mining --- .github/workflows/github-actions-es.yml | 23 -- integration_tests/common.go | 35 +- integration_tests/kzg_poseidon_prover_test.go | 63 ---- integration_tests/kzg_prover_test.go | 5 +- integration_tests/node_mine_test.go | 320 +++++++++++------- integration_tests/run_tests.sh | 5 +- integration_tests/zk_prover_test.go | 49 +++ 7 files changed, 262 insertions(+), 238 deletions(-) delete mode 100644 .github/workflows/github-actions-es.yml delete mode 100644 integration_tests/kzg_poseidon_prover_test.go diff --git a/.github/workflows/github-actions-es.yml b/.github/workflows/github-actions-es.yml deleted file mode 100644 index 16d105b7..00000000 --- a/.github/workflows/github-actions-es.yml +++ /dev/null @@ -1,23 +0,0 @@ -# This workflow will build a golang project -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go -name: GitHub Actions ES -run-name: ${{ github.actor }} is push code to main 🚀 -on: - push: - branches: [ "main" ] -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: '1.20' - - - name: Build - run: go build -v ./cmd/... - - - name: Test - run: go test -v ./... -tags ci diff --git a/integration_tests/common.go b/integration_tests/common.go index 1569fb52..3358a839 100644 --- a/integration_tests/common.go +++ b/integration_tests/common.go @@ -7,7 +7,6 @@ import ( "context" "fmt" "io" - "math/big" "net/http" "github.com/ethereum/go-ethereum" @@ -15,7 +14,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient" esLog "github.com/ethstorage/go-ethstorage/ethstorage/log" - "github.com/ethstorage/go-ethstorage/ethstorage/storage" ) const ( @@ -25,32 +23,13 @@ const ( ) var ( - contractAddr1GB = common.HexToAddress("0x24d29453B4B4917117B4af022Ad1B0a9aBc2c7DE") + contractAddr16kv = common.HexToAddress("0x192eE460Cf3A5AF51F9A7427Bb07237A2841D2d1") contractAddrDevnet2 = common.HexToAddress("0xb4B46bdAA835F8E4b4d8e208B6559cD267851051") minerAddr = common.HexToAddress("0x534632D6d7aD1fe5f832951c97FDe73E4eFD9a77") value = "1000000000000000" lg = esLog.NewLogger(esLog.DefaultCLIConfig()) ) -func initStorageConfig(ctx context.Context, client *ethclient.Client, l1Contract, miner common.Address) (*storage.StorageConfig, error) { - maxKvSizeBits, err := readUintFromContract(ctx, client, l1Contract, "maxKvSizeBits") - if err != nil { - return nil, err - } - chunkSizeBits := maxKvSizeBits - shardEntryBits, err := readUintFromContract(ctx, client, l1Contract, "shardEntryBits") - if err != nil { - return nil, err - } - return &storage.StorageConfig{ - L1Contract: l1Contract, - Miner: miner, - KvSize: 1 << maxKvSizeBits, - ChunkSize: 1 << chunkSizeBits, - KvEntriesPerShard: 1 << shardEntryBits, - }, nil -} - func readSlotFromContract(ctx context.Context, client *ethclient.Client, l1Contract common.Address, fieldName string) ([]byte, error) { h := crypto.Keccak256Hash([]byte(fieldName + "()")) msg := ethereum.CallMsg{ @@ -64,16 +43,6 @@ func readSlotFromContract(ctx context.Context, client *ethclient.Client, l1Contr return bs, nil } -func readUintFromContract(ctx context.Context, client *ethclient.Client, l1Contract common.Address, fieldName string) (uint64, error) { - bs, err := readSlotFromContract(ctx, client, l1Contract, fieldName) - if err != nil { - return 0, err - } - value := new(big.Int).SetBytes(bs).Uint64() - lg.Info("Read uint from contract", "field", fieldName, "value", value) - return value, nil -} - func callVerify(calldata []byte) error { ctx := context.Background() client, err := ethclient.DialContext(ctx, l1Endpoint) @@ -83,7 +52,7 @@ func callVerify(calldata []byte) error { } defer client.Close() msg := ethereum.CallMsg{ - To: &contractAddr1GB, + To: &contractAddr16kv, Data: calldata, } bs, err := client.CallContract(ctx, msg, nil) diff --git a/integration_tests/kzg_poseidon_prover_test.go b/integration_tests/kzg_poseidon_prover_test.go deleted file mode 100644 index 48ba5c1a..00000000 --- a/integration_tests/kzg_poseidon_prover_test.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2022-2023, EthStorage. -// For license information, see https://github.com/ethstorage/es-node/blob/main/LICENSE - -//go:build !ci - -package integration - -import ( - "os" - "path/filepath" - "testing" - - "github.com/ethereum/go-ethereum/common" - esLog "github.com/ethstorage/go-ethstorage/ethstorage/log" - "github.com/ethstorage/go-ethstorage/ethstorage/prover" -) - -func TestKZGPoseidonProver_GenerateZKProofs(t *testing.T) { - type args struct { - encodingKeys []common.Hash - chunkIdxes []uint64 - } - tests := []struct { - name string - args args - wantErr bool - }{ - { - name: "test 1", - args: args{ - []common.Hash{ - common.HexToHash("0x1"), - common.HexToHash("0x22222222222"), - common.HexToHash("0x1e88fb83944b20562a100533d0521b90bf7df7cc6e0aaa1c46482b67c7b370ab"), - }, - []uint64{0, 2222, 4095}, - }, - wantErr: false, - }, - } - - proverPath, _ := filepath.Abs(prPath) - zkeyFull := filepath.Join(proverPath, snarkLibDir, zkeyFile) - if _, err := os.Stat(zkeyFull); os.IsNotExist(err) { - t.Fatalf("%s not found", zkeyFull) - } - prv := prover.NewKZGPoseidonProver(proverPath, zkeyFile, esLog.NewLogger(esLog.DefaultCLIConfig())) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - proofs, masks, err := prv.GenerateZKProofs(tt.args.encodingKeys, tt.args.chunkIdxes) - if (err != nil) != tt.wantErr { - t.Errorf("GenerateZKProofs() error = %v, wantErr %v", err, tt.wantErr) - return - } - for i := 0; i < len(tt.args.chunkIdxes); i++ { - err = verifyDecodeSample(proofs[i], tt.args.chunkIdxes[i], tt.args.encodingKeys[i], masks[i]) - if (err != nil) != tt.wantErr { - t.Errorf("ZKProver.GenerateZKProofs() %d decodeSample err: %v", i, err) - } - } - }) - } -} diff --git a/integration_tests/kzg_prover_test.go b/integration_tests/kzg_prover_test.go index 595cd1e1..06d10ff9 100644 --- a/integration_tests/kzg_prover_test.go +++ b/integration_tests/kzg_prover_test.go @@ -96,7 +96,6 @@ func uploadBlob(t *testing.T, data []byte) common.Hash { blbKey := "0x0000000000000000000000000000000000000000000000000000000000000001" blbIdx := common.Big0 length := new(big.Int).SetInt64(128 * 1024) - // calldata := "0x4581a920000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000" h := crypto.Keccak256Hash([]byte("putBlob(bytes32,uint256,uint256)")) mid := h[0:4] @@ -116,7 +115,7 @@ func uploadBlob(t *testing.T, data []byte) common.Hash { valBig, _ := new(big.Int).SetString(value, 10) estimatedGas, err := client.EstimateGas(ctx, ethereum.CallMsg{ From: signer, - To: &contractAddr1GB, + To: &contractAddr16kv, Value: valBig, Data: calldata, }) @@ -127,7 +126,7 @@ func uploadBlob(t *testing.T, data []byte) common.Hash { tx := utils.SendBlobTx( l1Endpoint, - contractAddr1GB, + contractAddr16kv, pk, data, true, diff --git a/integration_tests/node_mine_test.go b/integration_tests/node_mine_test.go index f7c67b82..9c8c809b 100644 --- a/integration_tests/node_mine_test.go +++ b/integration_tests/node_mine_test.go @@ -9,22 +9,21 @@ import ( "context" "fmt" "io" + "io/ioutil" + "math/big" "net/http" "os" - "os/signal" "path/filepath" "strings" - "syscall" + "sync" "testing" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" "github.com/ethstorage/go-ethstorage/cmd/es-utils/utils" "github.com/ethstorage/go-ethstorage/ethstorage" - es "github.com/ethstorage/go-ethstorage/ethstorage" "github.com/ethstorage/go-ethstorage/ethstorage/eth" "github.com/ethstorage/go-ethstorage/ethstorage/miner" "github.com/ethstorage/go-ethstorage/ethstorage/p2p/protocol" @@ -38,90 +37,24 @@ const ( dataFileName = "shard-%d.dat" ) -var shardIds = []uint64{0} +var shardIds = []uint64{0, 1} -// to use a new contract and override data files: -// go test -run ^TestMining$ github.com/ethstorage/go-ethstorage/ethstorage/node -v count=1 -contract=0x9BE8dEbb712A3c0dD163Da85D3aF867793Aef3E6 -init=true func TestMining(t *testing.T) { - contract := contractAddr1GB - // contract := common.HexToAddress("0xf8928E47ab82912025FD45589f2444fF3a0FDa73") - flagContract := getArg("-contract") - if flagContract != "" { - contract = common.HexToAddress(flagContract) - lg.Info("Use the contract address from flag", "contract", flagContract) - } - init := false - flagInitStorage := getArg("-init") - if flagInitStorage == "true" { - lg.Info("Data files will be removed") - init = true - } - - intialized := false - existFile := 0 - for _, shardId := range shardIds { - fileName := fmt.Sprintf(dataFileName, shardId) - if _, err := os.Stat(fileName); !os.IsNotExist(err) { - if init { - os.Remove(fileName) - } else { - existFile++ - } - } - } - - if existFile == 2 { - intialized = true - lg.Info("Data files already exist, will start mining") - } - // if existFile == 1 { - // lg.Crit("One of the data files is missing, please check") - // } - if existFile == 0 { - lg.Info("Will initialize the data files, please make sure you use a new contract") - } + contract := contractAddr16kv pClient, err := eth.Dial(l1Endpoint, contract, lg) if err != nil { t.Fatalf("Failed to connect to the Ethereum client: %v", err) } - storConfig, err := initStorageConfig(context.Background(), pClient.Client, contract, minerAddr) + storConfig := initStorageConfig(t, pClient, contract, minerAddr) + files, err := CreateDataFiles(storConfig) if err != nil { - t.Fatalf("Failed to init storage client: %v", err) - } - var files []string - if !intialized { - f, err := CreateDataFiles(storConfig) - if err != nil { - t.Fatalf("Create data files error: %v", err) - } - files = f - } else { - for _, shardIdx := range shardIds { - fileName := fmt.Sprintf(dataFileName, shardIdx) - files = append(files, fileName) - } + t.Fatalf("Create data files error: %v", err) } storConfig.Filenames = files - if os.Getenv(pkName) == "" { - t.Fatal("No private key in env", pkName) - } - signerCfg := signer.CLIConfig{ - PrivateKey: os.Getenv(pkName), - } - factory, addrFrom, err := signer.SignerFactoryFromConfig(signerCfg) - if err != nil { - t.Fatal("SignerFactoryFromConfig err", err) - } - miningConfig := &miner.DefaultConfig - miningConfig.SignerFnFactory = factory - miningConfig.SignerAddr = addrFrom - zkWorkingDir, err := filepath.Abs("../prover") - if err != nil { - t.Fatalf("Get zkWorkingDir error: %v", err) - } - miningConfig.ZKWorkingDir = zkWorkingDir + miningConfig := initMiningConfig(t, contract, pClient) + defer cleanFiles(miningConfig.ZKWorkingDir) shardManager, err := initShardManager(*storConfig) if err != nil { t.Fatalf("init shard manager error: %v", err) @@ -132,9 +65,9 @@ func TestMining(t *testing.T) { feed := new(event.Feed) l1api := miner.NewL1MiningAPI(pClient, lg) - pvr := prover.NewKZGPoseidonProver(zkWorkingDir, "blob_poseidon.zkey", lg) + pvr := prover.NewKZGPoseidonProver(miningConfig.ZKWorkingDir, zkeyFile, lg) mnr := miner.New(miningConfig, storageManager, l1api, &pvr, feed, lg) - log.Info("Initialized miner") + lg.Info("Initialized miner") l1HeadsSub := event.ResubscribeErr(time.Second*10, func(ctx context.Context, err error) (event.Subscription, error) { if err != nil { @@ -148,31 +81,196 @@ func TestMining(t *testing.T) { } }) }) - if !intialized { - prepareData(t, pClient, storageManager) - // fillEmpty(t, pClient, storageManager) - } + go func() { + err, ok := <-l1HeadsSub.Err() + if !ok { + return + } + lg.Error("L1 heads subscription error", "err", err) + }() + // prepareData(t, pClient, storageManager) + fillEmpty(t, pClient, storageManager) mnr.Start() - feed.Send(protocol.EthStorageSyncDone{ - DoneType: protocol.SingleShardDone, - ShardId: 0, - }) - time.Sleep(360 * time.Second) - feed.Send(protocol.EthStorageSyncDone{ - DoneType: protocol.SingleShardDone, - ShardId: 1, - }) - c := make(chan os.Signal) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - <-c - mnr.Close() + var wg sync.WaitGroup + minedShardCh := make(chan uint64) + for _, s := range shardIds { + feed.Send(protocol.EthStorageSyncDone{ + DoneType: protocol.SingleShardDone, + ShardId: s, + }) + info, err := l1api.GetMiningInfo( + context.Background(), + contract, + s, + ) + if err != nil { + t.Fatalf("Failed to get es mining info for shard %d: %v", s, err) + } + go waitForMined(l1api, contract, mnr.ChainHeadCh, s, info.BlockMined.Uint64(), minedShardCh) + wg.Add(1) + time.Sleep(360 * time.Second) + } - l1HeadsSub.Unsubscribe() + for minedShard := range minedShardCh { + lg.Info("Mined shard", "shard", minedShard) + wg.Done() + } + wg.Wait() + mnr.Close() close() } -func initShardManager(storConfig storage.StorageConfig) (*es.ShardManager, error) { +func cleanFiles(proverDir string) { + for _, shardId := range shardIds { + fileName := fmt.Sprintf(dataFileName, shardId) + if _, err := os.Stat(fileName); !os.IsNotExist(err) { + err = os.Remove(fileName) + if err != nil { + fmt.Println(err) + } + } + } + + folderPath := filepath.Join(proverDir, "snarkbuild") + files, err := ioutil.ReadDir(folderPath) + if err != nil { + fmt.Println(err) + return + } + for _, file := range files { + if !strings.HasPrefix(file.Name(), ".") { + err = os.Remove(filepath.Join(folderPath, file.Name())) + if err != nil { + fmt.Println(err) + } + } + } +} + +func initMiningConfig(t *testing.T, l1Contract common.Address, client *eth.PollingClient) *miner.Config { + miningConfig := &miner.Config{} + pk := os.Getenv(pkName) + if pk == "" { + t.Fatal("No private key in env", pkName) + } + factory, addrFrom, err := signer.SignerFactoryFromConfig(signer.CLIConfig{ + PrivateKey: pk, + }) + if err != nil { + t.Fatal("SignerFactoryFromConfig err", err) + } + miningConfig.SignerFnFactory = factory + miningConfig.SignerAddr = addrFrom + result, err := client.ReadContractField("randomChecks") + if err != nil { + t.Fatal("get randomChecks", err) + } + miningConfig.RandomChecks = new(big.Int).SetBytes(result).Uint64() + result, err = client.ReadContractField("nonceLimit") + if err != nil { + t.Fatal("get nonceLimit", err) + } + miningConfig.NonceLimit = new(big.Int).SetBytes(result).Uint64() + result, err = client.ReadContractField("minimumDiff") + if err != nil { + t.Fatal("get minimumDiff", err) + } + miningConfig.MinimumDiff = new(big.Int).SetBytes(result) + result, err = client.ReadContractField("cutoff") + if err != nil { + t.Fatal("get cutoff", err) + } + miningConfig.Cutoff = new(big.Int).SetBytes(result) + result, err = client.ReadContractField("diffAdjDivisor") + if err != nil { + t.Fatal("get diffAdjDivisor", err) + } + miningConfig.DiffAdjDivisor = new(big.Int).SetBytes(result) + + result, err = client.ReadContractField("dcfFactor") + if err != nil { + t.Fatal("get dcfFactor", err) + } + miningConfig.DcfFactor = new(big.Int).SetBytes(result) + result, err = client.ReadContractField("startTime") + if err != nil { + t.Fatal("get startTime", err) + } + miningConfig.StartTime = new(big.Int).SetBytes(result).Uint64() + result, err = client.ReadContractField("shardEntryBits") + if err != nil { + t.Fatal("get shardEntryBits", err) + } + miningConfig.ShardEntry = 1 << new(big.Int).SetBytes(result).Uint64() + result, err = client.ReadContractField("treasuryShare") + if err != nil { + t.Fatal("get treasuryShare", err) + } + miningConfig.TreasuryShare = new(big.Int).SetBytes(result).Uint64() + result, err = client.ReadContractField("storageCost") + if err != nil { + t.Fatal("get storageCost", err) + } + miningConfig.StorageCost = new(big.Int).SetBytes(result) + result, err = client.ReadContractField("prepaidAmount") + if err != nil { + t.Fatal("get prepaidAmount", err) + } + miningConfig.PrepaidAmount = new(big.Int).SetBytes(result) + + proverPath, _ := filepath.Abs(prPath) + zkeyFull := filepath.Join(proverPath, snarkLibDir, zkeyFile) + if _, err := os.Stat(zkeyFull); os.IsNotExist(err) { + t.Fatalf("%s not found", zkeyFull) + } + miningConfig.ZKWorkingDir = proverPath + miningConfig.ThreadsPerShard = 2 + miningConfig.MinimumProfit = new(big.Int).SetInt64(-1e18) + return miningConfig +} + +func waitForMined(l1api miner.L1API, contract common.Address, chainHeadCh chan eth.L1BlockRef, shardIdx, lastMined uint64, exitCh chan uint64) { + for range chainHeadCh { + info, err := l1api.GetMiningInfo( + context.Background(), + contract, + shardIdx, + ) + if err != nil { + lg.Warn("Failed to get es mining info", "error", err.Error()) + continue + } + if info.BlockMined.Uint64() > lastMined { + lg.Info("Mined new", "shard", shardIdx, "lastMined", info.BlockMined) + exitCh <- shardIdx + return + } + } +} + +func initStorageConfig(t *testing.T, client *eth.PollingClient, l1Contract, miner common.Address) *storage.StorageConfig { + result, err := client.ReadContractField("maxKvSizeBits") + if err != nil { + t.Fatal("get maxKvSizeBits", err) + } + maxKvSizeBits := new(big.Int).SetBytes(result).Uint64() + chunkSizeBits := maxKvSizeBits + result, err = client.ReadContractField("shardEntryBits") + if err != nil { + t.Fatal("get shardEntryBits", err) + } + shardEntryBits := new(big.Int).SetBytes(result).Uint64() + return &storage.StorageConfig{ + L1Contract: l1Contract, + Miner: miner, + KvSize: 1 << maxKvSizeBits, + ChunkSize: 1 << chunkSizeBits, + KvEntriesPerShard: 1 << shardEntryBits, + } +} + +func initShardManager(storConfig storage.StorageConfig) (*ethstorage.ShardManager, error) { shardManager := ethstorage.NewShardManager(storConfig.L1Contract, storConfig.KvSize, storConfig.KvEntriesPerShard, storConfig.ChunkSize) for _, filename := range storConfig.Filenames { var err error @@ -195,20 +293,16 @@ func initShardManager(storConfig storage.StorageConfig) (*es.ShardManager, error } func fillEmpty(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage.StorageManager) { - empty := make([]byte, 0) - block, err := l1Client.BlockNumber(context.Background()) - if err != nil { - t.Fatalf("Failed to get block number %v", err) - } - storageMgr.Reset(int64(block)) - limit := storageMgr.KvEntries() * uint64(len(shardIds)) + lg.Info("Filling empty started") + totalBlobs := storageMgr.KvEntries() * uint64(len(shardIds)) lastBlobIdx := storageMgr.LastKvIndex() - for idx := lastBlobIdx; idx < limit; idx++ { - err = storageMgr.CommitBlob(idx, empty, common.Hash{}) - if err != nil { - t.Fatalf("write empty to kv file fail, index: %d; error: %s", idx, err.Error()) - } + lg.Info("Filling empty", "lastBlobIdx", lastBlobIdx, "totalBlobs", totalBlobs) + + inserted, next, err := storageMgr.CommitEmptyBlobs(lastBlobIdx, totalBlobs-1) + if err != nil { + t.Fatalf("Commit empty blobs failed %v", err) } + lg.Info("Filling empty done", "inserted", inserted, "next", next) } func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage.StorageManager) { @@ -285,11 +379,11 @@ func CreateDataFiles(cfg *storage.StorageConfig) ([]string, error) { chunkPerKv := cfg.KvSize / cfg.ChunkSize startChunkId := shardIdx * cfg.KvEntriesPerShard * chunkPerKv chunkIdxLen := chunkPerKv * cfg.KvEntriesPerShard - log.Info("Creating data file", "chunkIdxStart", startChunkId, "chunkIdxLen", chunkIdxLen, "chunkSize", cfg.ChunkSize, "miner", cfg.Miner, "encodeType", es.ENCODE_BLOB_POSEIDON) + lg.Info("Creating data file", "chunkIdxStart", startChunkId, "chunkIdxLen", chunkIdxLen, "chunkSize", cfg.ChunkSize, "miner", cfg.Miner, "encodeType", ethstorage.ENCODE_BLOB_POSEIDON) - df, err := es.Create(fileName, startChunkId, chunkPerKv*cfg.KvEntriesPerShard, 0, cfg.KvSize, es.ENCODE_BLOB_POSEIDON, cfg.Miner, cfg.ChunkSize) + df, err := ethstorage.Create(fileName, startChunkId, chunkPerKv*cfg.KvEntriesPerShard, 0, cfg.KvSize, ethstorage.ENCODE_BLOB_POSEIDON, cfg.Miner, cfg.ChunkSize) if err != nil { - log.Crit("Creating data file", "error", err) + lg.Crit("Creating data file", "error", err) } lg.Info("Data file created", "shard", shardIdx, "file", fileName, "datafile", fmt.Sprintf("%+v", df)) diff --git a/integration_tests/run_tests.sh b/integration_tests/run_tests.sh index b4db3f02..a86e0713 100755 --- a/integration_tests/run_tests.sh +++ b/integration_tests/run_tests.sh @@ -22,6 +22,5 @@ if [ ! -e ${zkey_file} ]; then curl -Lb ./cookie "https://drive.google.com/uc?export=download&`echo ${html}|grep -Eo 'confirm=[a-zA-Z0-9\-_]+'`&id=${file_id}" -o ${zkey_file} fi -# go test -timeout 0 github.com/ethstorage/go-ethstorage/integration_tests - -go test -timeout 0 -run ^TestKZGProver_GenerateKZGProof$ github.com/ethstorage/go-ethstorage/integration_tests +# go test -timeout 0 github.com/ethstorage/go-ethstorage/integration_tests -v +go test -timeout 0 -run ^TestMining$ github.com/ethstorage/go-ethstorage/integration_tests -v diff --git a/integration_tests/zk_prover_test.go b/integration_tests/zk_prover_test.go index a082caeb..6c58eb91 100644 --- a/integration_tests/zk_prover_test.go +++ b/integration_tests/zk_prover_test.go @@ -18,6 +18,7 @@ import ( "strings" "testing" + esLog "github.com/ethstorage/go-ethstorage/ethstorage/log" "github.com/ethstorage/go-ethstorage/ethstorage/prover" "github.com/consensys/gnark-crypto/ecc/bn254/fr" @@ -117,6 +118,54 @@ func TestZKProver_GenerateZKProof(t *testing.T) { } } +// prove in parallel +func TestKZGPoseidonProver_GenerateZKProofs(t *testing.T) { + type args struct { + encodingKeys []common.Hash + chunkIdxes []uint64 + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "test 1", + args: args{ + []common.Hash{ + common.HexToHash("0x1"), + common.HexToHash("0x22222222222"), + common.HexToHash("0x1e88fb83944b20562a100533d0521b90bf7df7cc6e0aaa1c46482b67c7b370ab"), + }, + []uint64{0, 2222, 4095}, + }, + wantErr: false, + }, + } + + proverPath, _ := filepath.Abs(prPath) + zkeyFull := filepath.Join(proverPath, snarkLibDir, zkeyFile) + if _, err := os.Stat(zkeyFull); os.IsNotExist(err) { + t.Fatalf("%s not found", zkeyFull) + } + prv := prover.NewKZGPoseidonProver(proverPath, zkeyFile, esLog.NewLogger(esLog.DefaultCLIConfig())) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proofs, masks, err := prv.GenerateZKProofs(tt.args.encodingKeys, tt.args.chunkIdxes) + if (err != nil) != tt.wantErr { + t.Errorf("GenerateZKProofs() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := 0; i < len(tt.args.chunkIdxes); i++ { + err = verifyDecodeSample(proofs[i], tt.args.chunkIdxes[i], tt.args.encodingKeys[i], masks[i]) + if (err != nil) != tt.wantErr { + t.Errorf("ZKProver.GenerateZKProofs() %d decodeSample err: %v", i, err) + } + } + }) + } +} + func GenerateMask(encodingKey common.Hash, chunkIdx uint64) (common.Hash, error) { if int(chunkIdx) >= eth.FieldElementsPerBlob { return common.Hash{}, fmt.Errorf("chunk index out of scope") From 353882428566f6e285acd14d7d660e37d200f589 Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 19 Dec 2023 19:04:09 +0800 Subject: [PATCH 14/14] fixes --- ethstorage/miner/l1_mining_api.go | 4 +- integration_tests/node_mine_test.go | 224 ++++++++++++++-------------- 2 files changed, 118 insertions(+), 110 deletions(-) diff --git a/ethstorage/miner/l1_mining_api.go b/ethstorage/miner/l1_mining_api.go index 12c74f21..55512a9e 100644 --- a/ethstorage/miner/l1_mining_api.go +++ b/ethstorage/miner/l1_mining_api.go @@ -222,7 +222,7 @@ func (m *l1MiningAPI) estimateReward(ctx context.Context, cfg Config, contract c } minedTs := curBlock.Time - (new(big.Int).Sub(curBlock.Number, block).Uint64())*12 - var reward *big.Int + reward := big.NewInt(0) if shard < lastShard { basePayment := new(big.Int).Mul(cfg.StorageCost, new(big.Int).SetUint64(cfg.ShardEntry)) reward = paymentIn(basePayment, cfg.DcfFactor, lastMineTime, minedTs, cfg.StartTime) @@ -236,7 +236,7 @@ func (m *l1MiningAPI) estimateReward(ctx context.Context, cfg Config, contract c } } minerReward := new(big.Int).Div( - new(big.Int).Mul(new(big.Int).SetUint64((rewardDenominator-cfg.TreasuryShare)), reward), + new(big.Int).Mul(new(big.Int).SetUint64(rewardDenominator-cfg.TreasuryShare), reward), new(big.Int).SetUint64(rewardDenominator), ) return minerReward, nil diff --git a/integration_tests/node_mine_test.go b/integration_tests/node_mine_test.go index 9c8c809b..a2667877 100644 --- a/integration_tests/node_mine_test.go +++ b/integration_tests/node_mine_test.go @@ -20,8 +20,8 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/rpc" "github.com/ethstorage/go-ethstorage/cmd/es-utils/utils" "github.com/ethstorage/go-ethstorage/ethstorage" "github.com/ethstorage/go-ethstorage/ethstorage/eth" @@ -46,8 +46,16 @@ func TestMining(t *testing.T) { if err != nil { t.Fatalf("Failed to connect to the Ethereum client: %v", err) } + + lastKv, err := pClient.GetStorageLastBlobIdx(rpc.LatestBlockNumber.Int64()) + if err != nil { + lg.Error("Failed to get lastKvIdx", "error", err) + } else { + lg.Info("lastKv", "lastKv", lastKv) + } + storConfig := initStorageConfig(t, pClient, contract, minerAddr) - files, err := CreateDataFiles(storConfig) + files, err := createDataFiles(storConfig) if err != nil { t.Fatalf("Create data files error: %v", err) } @@ -88,7 +96,7 @@ func TestMining(t *testing.T) { } lg.Error("L1 heads subscription error", "err", err) }() - // prepareData(t, pClient, storageManager) + // prepareData(t, pClient, storageManager, miningConfig.StorageCost.String()) fillEmpty(t, pClient, storageManager) mnr.Start() @@ -112,11 +120,20 @@ func TestMining(t *testing.T) { time.Sleep(360 * time.Second) } - for minedShard := range minedShardCh { - lg.Info("Mined shard", "shard", minedShard) - wg.Done() - } + go func() { + minedShards := make(map[uint64]bool) + for minedShard := range minedShardCh { + if !minedShards[minedShard] { + lg.Info("Mined shard", "shard", minedShard) + minedShards[minedShard] = true + wg.Done() + lg.Info("wait group done") + } + } + }() + lg.Info("wait group waiting") wg.Wait() + l1HeadsSub.Unsubscribe() mnr.Close() close() } @@ -140,7 +157,7 @@ func cleanFiles(proverDir string) { } for _, file := range files { if !strings.HasPrefix(file.Name(), ".") { - err = os.Remove(filepath.Join(folderPath, file.Name())) + err = os.RemoveAll(filepath.Join(folderPath, file.Name())) if err != nil { fmt.Println(err) } @@ -148,88 +165,6 @@ func cleanFiles(proverDir string) { } } -func initMiningConfig(t *testing.T, l1Contract common.Address, client *eth.PollingClient) *miner.Config { - miningConfig := &miner.Config{} - pk := os.Getenv(pkName) - if pk == "" { - t.Fatal("No private key in env", pkName) - } - factory, addrFrom, err := signer.SignerFactoryFromConfig(signer.CLIConfig{ - PrivateKey: pk, - }) - if err != nil { - t.Fatal("SignerFactoryFromConfig err", err) - } - miningConfig.SignerFnFactory = factory - miningConfig.SignerAddr = addrFrom - result, err := client.ReadContractField("randomChecks") - if err != nil { - t.Fatal("get randomChecks", err) - } - miningConfig.RandomChecks = new(big.Int).SetBytes(result).Uint64() - result, err = client.ReadContractField("nonceLimit") - if err != nil { - t.Fatal("get nonceLimit", err) - } - miningConfig.NonceLimit = new(big.Int).SetBytes(result).Uint64() - result, err = client.ReadContractField("minimumDiff") - if err != nil { - t.Fatal("get minimumDiff", err) - } - miningConfig.MinimumDiff = new(big.Int).SetBytes(result) - result, err = client.ReadContractField("cutoff") - if err != nil { - t.Fatal("get cutoff", err) - } - miningConfig.Cutoff = new(big.Int).SetBytes(result) - result, err = client.ReadContractField("diffAdjDivisor") - if err != nil { - t.Fatal("get diffAdjDivisor", err) - } - miningConfig.DiffAdjDivisor = new(big.Int).SetBytes(result) - - result, err = client.ReadContractField("dcfFactor") - if err != nil { - t.Fatal("get dcfFactor", err) - } - miningConfig.DcfFactor = new(big.Int).SetBytes(result) - result, err = client.ReadContractField("startTime") - if err != nil { - t.Fatal("get startTime", err) - } - miningConfig.StartTime = new(big.Int).SetBytes(result).Uint64() - result, err = client.ReadContractField("shardEntryBits") - if err != nil { - t.Fatal("get shardEntryBits", err) - } - miningConfig.ShardEntry = 1 << new(big.Int).SetBytes(result).Uint64() - result, err = client.ReadContractField("treasuryShare") - if err != nil { - t.Fatal("get treasuryShare", err) - } - miningConfig.TreasuryShare = new(big.Int).SetBytes(result).Uint64() - result, err = client.ReadContractField("storageCost") - if err != nil { - t.Fatal("get storageCost", err) - } - miningConfig.StorageCost = new(big.Int).SetBytes(result) - result, err = client.ReadContractField("prepaidAmount") - if err != nil { - t.Fatal("get prepaidAmount", err) - } - miningConfig.PrepaidAmount = new(big.Int).SetBytes(result) - - proverPath, _ := filepath.Abs(prPath) - zkeyFull := filepath.Join(proverPath, snarkLibDir, zkeyFile) - if _, err := os.Stat(zkeyFull); os.IsNotExist(err) { - t.Fatalf("%s not found", zkeyFull) - } - miningConfig.ZKWorkingDir = proverPath - miningConfig.ThreadsPerShard = 2 - miningConfig.MinimumProfit = new(big.Int).SetInt64(-1e18) - return miningConfig -} - func waitForMined(l1api miner.L1API, contract common.Address, chainHeadCh chan eth.L1BlockRef, shardIdx, lastMined uint64, exitCh chan uint64) { for range chainHeadCh { info, err := l1api.GetMiningInfo( @@ -242,7 +177,7 @@ func waitForMined(l1api miner.L1API, contract common.Address, chainHeadCh chan e continue } if info.BlockMined.Uint64() > lastMined { - lg.Info("Mined new", "shard", shardIdx, "lastMined", info.BlockMined) + lg.Info("Mined new", "shard", shardIdx, "lastMined", lastMined, "justMined", info.BlockMined) exitCh <- shardIdx return } @@ -295,17 +230,17 @@ func initShardManager(storConfig storage.StorageConfig) (*ethstorage.ShardManage func fillEmpty(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage.StorageManager) { lg.Info("Filling empty started") totalBlobs := storageMgr.KvEntries() * uint64(len(shardIds)) - lastBlobIdx := storageMgr.LastKvIndex() - lg.Info("Filling empty", "lastBlobIdx", lastBlobIdx, "totalBlobs", totalBlobs) + lastKvIdx := storageMgr.LastKvIndex() + lg.Info("Filling empty", "lastBlobIdx", lastKvIdx, "totalBlobs", totalBlobs) - inserted, next, err := storageMgr.CommitEmptyBlobs(lastBlobIdx, totalBlobs-1) + inserted, next, err := storageMgr.CommitEmptyBlobs(lastKvIdx, totalBlobs-1) if err != nil { t.Fatalf("Commit empty blobs failed %v", err) } lg.Info("Filling empty done", "inserted", inserted, "next", next) } -func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage.StorageManager) { +func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage.StorageManager, value string) { data, err := getSourceData() if err != nil { t.Fatalf("Get source data failed %v", err) @@ -315,7 +250,6 @@ func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstora var hashs []common.Hash var ids []uint64 - value := hexutil.EncodeUint64(500000000000000) txs := len(blobs) / maxBlobsPerTx last := len(blobs) % maxBlobsPerTx if last > 0 { @@ -355,7 +289,11 @@ func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstora t.Fatalf("Failed to get block number %v", err) } storageMgr.Reset(int64(block)) - for i := 0; i < len(shardIds)*int(storageMgr.KvEntries()); i++ { + limit := len(shardIds) * int(storageMgr.KvEntries()) + if limit > len(ids) { + limit = len(ids) + } + for i := 0; i < limit; i++ { err := storageMgr.CommitBlob(ids[i], blobs[i][:], hashs[i]) if err != nil { t.Fatalf("Failed to commit blob: id %d, error: %v", ids[i], err) @@ -363,12 +301,12 @@ func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstora } } -func CreateDataFiles(cfg *storage.StorageConfig) ([]string, error) { +func createDataFiles(cfg *storage.StorageConfig) ([]string, error) { var files []string for _, shardIdx := range shardIds { fileName := fmt.Sprintf(dataFileName, shardIdx) if _, err := os.Stat(fileName); err == nil { - lg.Crit("Creating data file", "error", "file already exists, will not overwrite", "file", fileName) + lg.Warn("Creating data file: file already exists, will be overwritten", "file", fileName) } if cfg.ChunkSize == 0 { lg.Crit("Creating data file", "error", "chunk size should not be 0") @@ -406,14 +344,84 @@ func getSourceData() ([]byte, error) { return data, nil } -func getArg(paramName string) string { - for _, arg := range os.Args { - pair := strings.Split(arg, "=") - if len(pair) == 2 && pair[0] == paramName { - paramValue := pair[1] - fmt.Printf("%s=%s\n", paramName, paramValue) - return paramValue - } +func initMiningConfig(t *testing.T, l1Contract common.Address, client *eth.PollingClient) *miner.Config { + miningConfig := &miner.Config{} + pk := os.Getenv(pkName) + if pk == "" { + t.Fatal("No private key in env", pkName) + } + factory, addrFrom, err := signer.SignerFactoryFromConfig(signer.CLIConfig{ + PrivateKey: pk, + }) + if err != nil { + t.Fatal("SignerFactoryFromConfig err", err) + } + miningConfig.SignerFnFactory = factory + miningConfig.SignerAddr = addrFrom + result, err := client.ReadContractField("randomChecks") + if err != nil { + t.Fatal("get randomChecks", err) + } + miningConfig.RandomChecks = new(big.Int).SetBytes(result).Uint64() + result, err = client.ReadContractField("nonceLimit") + if err != nil { + t.Fatal("get nonceLimit", err) + } + miningConfig.NonceLimit = new(big.Int).SetBytes(result).Uint64() + result, err = client.ReadContractField("minimumDiff") + if err != nil { + t.Fatal("get minimumDiff", err) + } + miningConfig.MinimumDiff = new(big.Int).SetBytes(result) + result, err = client.ReadContractField("cutoff") + if err != nil { + t.Fatal("get cutoff", err) + } + miningConfig.Cutoff = new(big.Int).SetBytes(result) + result, err = client.ReadContractField("diffAdjDivisor") + if err != nil { + t.Fatal("get diffAdjDivisor", err) + } + miningConfig.DiffAdjDivisor = new(big.Int).SetBytes(result) + + result, err = client.ReadContractField("dcfFactor") + if err != nil { + t.Fatal("get dcfFactor", err) + } + miningConfig.DcfFactor = new(big.Int).SetBytes(result) + result, err = client.ReadContractField("startTime") + if err != nil { + t.Fatal("get startTime", err) + } + miningConfig.StartTime = new(big.Int).SetBytes(result).Uint64() + result, err = client.ReadContractField("shardEntryBits") + if err != nil { + t.Fatal("get shardEntryBits", err) + } + miningConfig.ShardEntry = 1 << new(big.Int).SetBytes(result).Uint64() + result, err = client.ReadContractField("treasuryShare") + if err != nil { + t.Fatal("get treasuryShare", err) } - return "" + miningConfig.TreasuryShare = new(big.Int).SetBytes(result).Uint64() + result, err = client.ReadContractField("storageCost") + if err != nil { + t.Fatal("get storageCost", err) + } + miningConfig.StorageCost = new(big.Int).SetBytes(result) + result, err = client.ReadContractField("prepaidAmount") + if err != nil { + t.Fatal("get prepaidAmount", err) + } + miningConfig.PrepaidAmount = new(big.Int).SetBytes(result) + + proverPath, _ := filepath.Abs(prPath) + zkeyFull := filepath.Join(proverPath, snarkLibDir, zkeyFile) + if _, err := os.Stat(zkeyFull); os.IsNotExist(err) { + t.Fatalf("%s not found", zkeyFull) + } + miningConfig.ZKWorkingDir = proverPath + miningConfig.ThreadsPerShard = 2 + miningConfig.MinimumProfit = new(big.Int).SetInt64(-1e18) + return miningConfig }