From f0fc6e63850d6ddd38bc0e843c6116b9a6b9b9e9 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Thu, 8 Dec 2022 14:18:19 +0800 Subject: [PATCH 01/31] fix test --- integration_tests/configs/cronos-experimental-devnet.jsonnet | 1 + integration_tests/configs/disable_auto_deployment.jsonnet | 1 + 2 files changed, 2 insertions(+) diff --git a/integration_tests/configs/cronos-experimental-devnet.jsonnet b/integration_tests/configs/cronos-experimental-devnet.jsonnet index 6e6a2e7faf..630c7e29d4 100644 --- a/integration_tests/configs/cronos-experimental-devnet.jsonnet +++ b/integration_tests/configs/cronos-experimental-devnet.jsonnet @@ -12,6 +12,7 @@ config { address: '0.0.0.0:{EVMRPC_PORT}', 'ws-address': '0.0.0.0:{EVMRPC_PORT_WS}', }, + store:: super.store, }, genesis+: { app_state+: { diff --git a/integration_tests/configs/disable_auto_deployment.jsonnet b/integration_tests/configs/disable_auto_deployment.jsonnet index c3c47c3388..1bc6c6857f 100644 --- a/integration_tests/configs/disable_auto_deployment.jsonnet +++ b/integration_tests/configs/disable_auto_deployment.jsonnet @@ -9,6 +9,7 @@ config { 'json-rpc'+: { api:: super['api'], }, + store:: super.store, }, accounts: [{ name: 'community', From 66ee5978e188303dc01039975c9fe7c4c5a6729f Mon Sep 17 00:00:00 2001 From: mmsqe Date: Mon, 17 Oct 2022 09:34:51 +0800 Subject: [PATCH 02/31] test decode --- versiondb/backend_test_utils.go | 55 ++++++++++++++++++++++++--------- versiondb/tmdb/store_test.go | 43 ++++++++++++++++++++++++-- 2 files changed, 81 insertions(+), 17 deletions(-) diff --git a/versiondb/backend_test_utils.go b/versiondb/backend_test_utils.go index 507ab87ec8..a0ad857b8e 100644 --- a/versiondb/backend_test_utils.go +++ b/versiondb/backend_test_utils.go @@ -4,6 +4,7 @@ import ( "fmt" "testing" + "github.com/golang/protobuf/proto" "github.com/stretchr/testify/require" dbm "github.com/tendermint/tm-db" @@ -16,37 +17,46 @@ var ( key1Subkey = []byte("key1/subkey") ) -func SetupTestDB(t *testing.T, store VersionStore) { - changeSets := [][]types.StoreKVPair{ - { +func MockStoreKVPairs(v int64) []types.StoreKVPair { + switch v { + case 0: + return []types.StoreKVPair{ {StoreKey: "evm", Key: []byte("delete-in-block2"), Value: []byte("1")}, {StoreKey: "evm", Key: []byte("re-add-in-block3"), Value: []byte("1")}, {StoreKey: "evm", Key: []byte("z-genesis-only"), Value: []byte("2")}, {StoreKey: "evm", Key: []byte("modify-in-block2"), Value: []byte("1")}, {StoreKey: "staking", Key: []byte("key1"), Value: []byte("value1")}, {StoreKey: "staking", Key: []byte("key1/subkey"), Value: []byte("value1")}, - }, - { + } + case 1: + return []types.StoreKVPair{ {StoreKey: "evm", Key: []byte("re-add-in-block3"), Delete: true}, {StoreKey: "evm", Key: []byte("add-in-block1"), Value: []byte("1")}, {StoreKey: "staking", Key: []byte("key1"), Delete: true}, - }, - { + } + case 2: + return []types.StoreKVPair{ {StoreKey: "evm", Key: []byte("add-in-block2"), Value: []byte("1")}, {StoreKey: "evm", Key: []byte("delete-in-block2"), Delete: true}, {StoreKey: "evm", Key: []byte("modify-in-block2"), Value: []byte("2")}, {StoreKey: "evm", Key: []byte("key2"), Delete: true}, {StoreKey: "staking", Key: []byte("key1"), Value: []byte("value2")}, - }, - { + } + case 3: + return []types.StoreKVPair{ {StoreKey: "evm", Key: []byte("re-add-in-block3"), Value: []byte("2")}, - }, - { + } + case 4: + return []types.StoreKVPair{ {StoreKey: "evm", Key: []byte("re-add-in-block3"), Delete: true}, - }, + } } - for i, changeSet := range changeSets { - require.NoError(t, store.PutAtVersion(int64(i), changeSet)) + return nil +} + +func SetupTestDB(t *testing.T, store VersionStore) { + for i := 0; i < 5; i++ { + require.NoError(t, store.PutAtVersion(int64(i), MockStoreKVPairs(int64(i)))) } } @@ -273,3 +283,20 @@ func reversed[S ~[]E, E any](s S) []E { } return r } + +func TestDecodeData(t *testing.T, data []byte) (pairs []types.StoreKVPair) { + offset := 0 + for offset < len(data) { + size, n := proto.DecodeVarint(data[offset:]) + offset += n + pair := new(types.StoreKVPair) + err := proto.Unmarshal(data[offset:offset+int(size)], pair) + if err != nil { + t.Error(err) + return + } + pairs = append(pairs, *pair) + offset += int(size) + } + return +} diff --git a/versiondb/tmdb/store_test.go b/versiondb/tmdb/store_test.go index 24453198ff..40f49afd7b 100644 --- a/versiondb/tmdb/store_test.go +++ b/versiondb/tmdb/store_test.go @@ -1,14 +1,51 @@ package tmdb import ( + "bytes" "testing" + "github.com/cosmos/cosmos-sdk/codec" + codectypes "github.com/cosmos/cosmos-sdk/codec/types" "github.com/crypto-org-chain/cronos/versiondb" + "github.com/crypto-org-chain/cronos/x/cronos/types" + "github.com/stretchr/testify/require" dbm "github.com/tendermint/tm-db" ) +func storeCreator() versiondb.VersionStore { + return NewStore(dbm.NewMemDB(), dbm.NewMemDB(), dbm.NewMemDB()) +} + func TestTMDB(t *testing.T) { - versiondb.Run(t, func() versiondb.VersionStore { - return NewStore(dbm.NewMemDB(), dbm.NewMemDB(), dbm.NewMemDB()) - }) + versiondb.Run(t, storeCreator) +} + +func TestFeed(t *testing.T) { + registry := codectypes.NewInterfaceRegistry() + types.RegisterInterfaces(registry) + cdc := codec.NewProtoCodec(registry) + for i := 0; i < 4; i++ { + buf := new(bytes.Buffer) + v := int64(i) + storePairs := versiondb.MockStoreKVPairs(v) + for i := range storePairs { + bz, err := cdc.MarshalLengthPrefixed(&storePairs[i]) + require.NoError(t, err) + _, err = buf.Write(bz) + require.NoError(t, err) + } + pairs := versiondb.TestDecodeData(t, buf.Bytes()) + require.NotEmpty(t, pairs) + store := storeCreator() + require.NoError(t, store.PutAtVersion(v, pairs)) + for _, pair := range pairs { + value, err := store.GetAtVersion(pair.StoreKey, pair.Key, &v) + if pair.Delete { + require.Nil(t, value) + } else { + require.NotNil(t, value) + } + require.NoError(t, err) + } + } } From ec465d5c8b8f2d18a2747b8cc8630c42b2521447 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Mon, 17 Oct 2022 09:53:38 +0800 Subject: [PATCH 03/31] test query only --- integration_tests/configs/primary.jsonnet | 9 +++++ integration_tests/configs/replica.jsonnet | 12 ++++++ integration_tests/test_query_only.py | 45 +++++++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 integration_tests/configs/primary.jsonnet create mode 100644 integration_tests/configs/replica.jsonnet create mode 100644 integration_tests/test_query_only.py diff --git a/integration_tests/configs/primary.jsonnet b/integration_tests/configs/primary.jsonnet new file mode 100644 index 0000000000..9f3674aed8 --- /dev/null +++ b/integration_tests/configs/primary.jsonnet @@ -0,0 +1,9 @@ +local config = import 'default.jsonnet'; + +config { + 'cronos_777-1'+: { + 'app-config'+: { + 'minimum-gas-prices': '100000000000basetcro', + }, + }, +} diff --git a/integration_tests/configs/replica.jsonnet b/integration_tests/configs/replica.jsonnet new file mode 100644 index 0000000000..45c4fb95f1 --- /dev/null +++ b/integration_tests/configs/replica.jsonnet @@ -0,0 +1,12 @@ +local config = import 'primary.jsonnet'; + +config { + 'cronos_777-1'+: { + 'app-config'+: { + 'json-rpc': { + enable: false, + }, + 'grpc-only': true, + }, + }, +} diff --git a/integration_tests/test_query_only.py b/integration_tests/test_query_only.py new file mode 100644 index 0000000000..797df5ec56 --- /dev/null +++ b/integration_tests/test_query_only.py @@ -0,0 +1,45 @@ +from pathlib import Path + +import pytest + +from .network import setup_custom_cronos +from .utils import ( + ADDRS, +) + +@pytest.fixture(scope="module") +def primary(tmp_path_factory): + path = tmp_path_factory.mktemp("cronos-primary") + yield from setup_custom_cronos( + path, 26750, Path(__file__).parent / "configs/primary.jsonnet" + ) + + +@pytest.fixture(scope="module") +def replica(tmp_path_factory): + path = tmp_path_factory.mktemp("cronos-replica") + yield from setup_custom_cronos( + path, 26770, Path(__file__).parent / "configs/replica.jsonnet" + ) + + +def test_basic(primary, replica): + pw3 = primary.w3 + rw3 = replica.w3 + validator = ADDRS["validator"] + community = ADDRS["community"] + def print_balance(): + print(pw3.eth.get_balance(community)) + # print(rw3.eth.get_balance(community)) + + print_balance() + txhash = pw3.eth.send_transaction( + { + "from": validator, + "to": community, + "value": 1000, + } + ) + receipt = pw3.eth.wait_for_transaction_receipt(txhash) + assert receipt.status == 1 + print_balance() From 79163ab1b1aab3776e382c80a3fc6442ef3042be Mon Sep 17 00:00:00 2001 From: mmsqe Date: Mon, 17 Oct 2022 09:56:49 +0800 Subject: [PATCH 04/31] impl watch --- app/app.go | 38 ++++++++- file/decoder.go | 21 +++++ file/watcher.go | 147 ++++++++++++++++++++++++++++++++ file/watcher_test.go | 78 +++++++++++++++++ versiondb/backend_test_utils.go | 18 ---- versiondb/tmdb/store_test.go | 4 +- 6 files changed, 283 insertions(+), 23 deletions(-) create mode 100644 file/decoder.go create mode 100644 file/watcher.go create mode 100644 file/watcher_test.go diff --git a/app/app.go b/app/app.go index e9018efa04..0cbc63cc5a 100644 --- a/app/app.go +++ b/app/app.go @@ -6,6 +6,7 @@ import ( "net/http" "os" "path/filepath" + "time" "github.com/crypto-org-chain/cronos/x/cronos" "github.com/crypto-org-chain/cronos/x/cronos/middleware" @@ -22,6 +23,7 @@ import ( dbm "github.com/tendermint/tm-db" "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/client/grpc/tmservice" "github.com/cosmos/cosmos-sdk/codec" "github.com/cosmos/cosmos-sdk/server/api" @@ -123,6 +125,7 @@ import ( // this line is used by starport scaffolding # stargate/app/moduleImport + cronosfile "github.com/crypto-org-chain/cronos/file" "github.com/crypto-org-chain/cronos/versiondb" "github.com/crypto-org-chain/cronos/versiondb/tmdb" cronosclient "github.com/crypto-org-chain/cronos/x/cronos/client" @@ -353,12 +356,11 @@ func New( os.Exit(1) } - // configure state listening capabilities using AppOptions - // we are doing nothing with the returned streamingServices and waitGroup in this case streamers := cast.ToStringSlice(appOpts.Get("store.streamers")) for _, streamerName := range streamers { if streamerName == "versiondb" { - dataDir := filepath.Join(homePath, "data", "versiondb") + rootDir := cast.ToString(appOpts.Get(flags.FlagHome)) + dataDir := filepath.Join(rootDir, "data", "versiondb") if err := os.MkdirAll(dataDir, os.ModePerm); err != nil { panic(err) } @@ -376,6 +378,35 @@ func New( panic(err) } versionDB := tmdb.NewStore(plainDB, historyDB, changesetDB) + directory := filepath.Join(rootDir, "data", "file_streamer") + watcher := cronosfile.NewBlockFileWatcher(func(blockNum int64) string { + return cronosfile.GetLocalDataFileName(directory, blockNum) + }, true) + watcher.Start(1, time.Microsecond) + go func() { + // max retry for temporary io error + const maxRetry = 3 + retry := 0 + chData, chErr := watcher.SubscribeData(), watcher.SubscribeError() + for { + select { + case data := <-chData: + pairs, err := cronosfile.DecodeData(data.Data) + if err != nil { + panic(err) + } + retry = 0 + fmt.Printf("mm-pairs: %+v\n", pairs) + versionDB.PutAtVersion(data.BlockNum, pairs) + case err := <-chErr: + retry++ + if retry == maxRetry { + // data corrupt + panic(err) + } + } + } + }() // default to exposing all exposeStoreKeys := make([]storetypes.StoreKey, 0, len(keys)) @@ -388,7 +419,6 @@ func New( qms.MountTransientStores(tkeys) qms.MountMemoryStores(memKeys) bApp.SetQueryMultiStore(qms) - break } } diff --git a/file/decoder.go b/file/decoder.go new file mode 100644 index 0000000000..150d1f0be4 --- /dev/null +++ b/file/decoder.go @@ -0,0 +1,21 @@ +package file + +import ( + "github.com/cosmos/cosmos-sdk/store/types" + "github.com/gogo/protobuf/proto" +) + +func DecodeData(data []byte) (pairs []types.StoreKVPair, err error) { + offset := 0 + for offset < len(data) { + size, n := proto.DecodeVarint(data[offset:]) + offset += n + pair := new(types.StoreKVPair) + if err := proto.Unmarshal(data[offset:offset+int(size)], pair); err != nil { + return nil, err + } + pairs = append(pairs, *pair) + offset += int(size) + } + return +} diff --git a/file/watcher.go b/file/watcher.go new file mode 100644 index 0000000000..5bdc0e4fc8 --- /dev/null +++ b/file/watcher.go @@ -0,0 +1,147 @@ +package file + +import ( + "errors" + "fmt" + "io/ioutil" + "net/http" + "os" + "sync" + "time" +) + +var errNotExist = errors.New("file not exist") + +type fileDownloader interface { + GetData(path string) ([]byte, error) +} + +type localFileDownloader struct{} + +func (d *localFileDownloader) GetData(path string) ([]byte, error) { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + err = errNotExist + } + return nil, err + } + return data, nil +} + +type httpFileDownloader struct{} + +func (d *httpFileDownloader) GetData(path string) ([]byte, error) { + resp, err := http.Get(path) //nolint + if err != nil { + return nil, err + } + + defer resp.Body.Close() + + if resp.StatusCode != 200 { + if resp.StatusCode == http.StatusNotFound { + return nil, errNotExist + } + return nil, errors.New(resp.Status) + } + return ioutil.ReadAll(resp.Body) +} + +type BlockData struct { + BlockNum int64 + Data []byte +} + +type BlockFileWatcher struct { + getFilePath func(blockNum int64) string + downloader fileDownloader + chData chan *BlockData + chError chan error + chDone chan bool + startLock *sync.Mutex +} + +func NewBlockFileWatcher( + getFilePath func(blockNum int64) string, + isLocal bool, +) *BlockFileWatcher { + w := &BlockFileWatcher{ + getFilePath: getFilePath, + chData: make(chan *BlockData), + chError: make(chan error), + startLock: new(sync.Mutex), + } + if isLocal { + w.downloader = new(localFileDownloader) + } else { + w.downloader = new(httpFileDownloader) + } + return w +} + +func DataFileName(blockNum int64) string { + return fmt.Sprintf("block-%d-data", blockNum) +} + +func GetLocalDataFileName(directory string, blockNum int64) string { + return fmt.Sprintf("%s/%s", directory, DataFileName(blockNum)) +} + +func (w *BlockFileWatcher) SubscribeData() <-chan *BlockData { + return w.chData +} + +func (w *BlockFileWatcher) SubscribeError() <-chan error { + return w.chError +} + +func (w *BlockFileWatcher) Start( + blockNum int64, + interval time.Duration, +) { + w.startLock.Lock() + defer w.startLock.Unlock() + if w.chDone != nil { + return + } + + w.chDone = make(chan bool) + go func() { + for { + select { + case <-w.chDone: + return + + default: + path := w.getFilePath(blockNum) + data, err := w.downloader.GetData(path) + if err != nil { + if err != errNotExist { + // avoid blocked by error when not subscribe + select { + case w.chError <- err: + default: + } + } + } else { + w.chData <- &BlockData{ + BlockNum: blockNum, + Data: data, + } + blockNum++ + } + time.Sleep(interval) + } + } + }() +} + +func (w *BlockFileWatcher) Close() { + w.startLock.Lock() + defer w.startLock.Unlock() + if w.chDone != nil { + close(w.chDone) + w.chDone = nil + } +} diff --git a/file/watcher_test.go b/file/watcher_test.go new file mode 100644 index 0000000000..a5f4b2902c --- /dev/null +++ b/file/watcher_test.go @@ -0,0 +1,78 @@ +package file + +import ( + "fmt" + "log" + "net/http" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func setupDirectory(t *testing.T, directory string) func(t *testing.T) { + err := os.MkdirAll(directory, os.ModePerm) + require.NoError(t, err) + fmt.Println("setup directory:", directory) + return func(t *testing.T) { + os.RemoveAll(directory) + fmt.Println("cleanup directory") + } +} + +func setupBlockFiles(directory string, start, end int64) { + for i := start; i <= end; i++ { + file := GetLocalDataFileName(directory, i) + os.WriteFile(file, []byte(fmt.Sprint("block", i)), 0644) + } +} + +func start(watcher *BlockFileWatcher, endBlockNum int64) int64 { + watcher.Start(1, time.Microsecond) + counter := int64(0) + for data := range watcher.SubscribeData() { + if data != nil && len(data.Data) > 0 { + counter++ + } + if data.BlockNum == endBlockNum { + return counter + } + } + return counter +} + +func TestFileWatcher(t *testing.T) { + directory := "tmp" + teardown := setupDirectory(t, directory) + startBlockNum := int64(1) + endBlockNum := int64(2) + + defer teardown(t) + + t.Run("when sync via local", func(t *testing.T) { + setupBlockFiles(directory, startBlockNum, endBlockNum) + watcher := NewBlockFileWatcher(func(blockNum int64) string { + return GetLocalDataFileName(directory, blockNum) + }, true) + total := start(watcher, endBlockNum) + expected := endBlockNum - startBlockNum + 1 + require.Equal(t, total, expected) + }) + + t.Run("when sync via http", func(t *testing.T) { + setupBlockFiles(directory, startBlockNum, endBlockNum) + http.Handle("/", http.FileServer(http.Dir(directory))) + port := "8080" + fmt.Printf("Serving %s on HTTP port: %s\n", directory, port) + go func() { + log.Fatal(http.ListenAndServe(":"+port, nil)) + }() + watcher := NewBlockFileWatcher(func(blockNum int64) string { + return fmt.Sprintf("http://localhost:%s/%s", port, DataFileName(blockNum)) + }, false) + total := start(watcher, endBlockNum) + expected := endBlockNum - startBlockNum + 1 + require.Equal(t, total, expected) + }) +} diff --git a/versiondb/backend_test_utils.go b/versiondb/backend_test_utils.go index a0ad857b8e..a20f6caacc 100644 --- a/versiondb/backend_test_utils.go +++ b/versiondb/backend_test_utils.go @@ -4,7 +4,6 @@ import ( "fmt" "testing" - "github.com/golang/protobuf/proto" "github.com/stretchr/testify/require" dbm "github.com/tendermint/tm-db" @@ -283,20 +282,3 @@ func reversed[S ~[]E, E any](s S) []E { } return r } - -func TestDecodeData(t *testing.T, data []byte) (pairs []types.StoreKVPair) { - offset := 0 - for offset < len(data) { - size, n := proto.DecodeVarint(data[offset:]) - offset += n - pair := new(types.StoreKVPair) - err := proto.Unmarshal(data[offset:offset+int(size)], pair) - if err != nil { - t.Error(err) - return - } - pairs = append(pairs, *pair) - offset += int(size) - } - return -} diff --git a/versiondb/tmdb/store_test.go b/versiondb/tmdb/store_test.go index 40f49afd7b..ea7aee0b38 100644 --- a/versiondb/tmdb/store_test.go +++ b/versiondb/tmdb/store_test.go @@ -6,6 +6,7 @@ import ( "github.com/cosmos/cosmos-sdk/codec" codectypes "github.com/cosmos/cosmos-sdk/codec/types" + "github.com/crypto-org-chain/cronos/file" "github.com/crypto-org-chain/cronos/versiondb" "github.com/crypto-org-chain/cronos/x/cronos/types" "github.com/stretchr/testify/require" @@ -34,7 +35,8 @@ func TestFeed(t *testing.T) { _, err = buf.Write(bz) require.NoError(t, err) } - pairs := versiondb.TestDecodeData(t, buf.Bytes()) + pairs, err := file.DecodeData(buf.Bytes()) + require.NoError(t, err) require.NotEmpty(t, pairs) store := storeCreator() require.NoError(t, store.PutAtVersion(v, pairs)) From 90bb2f741fd9a006b7627ac394a293cfe03d246e Mon Sep 17 00:00:00 2001 From: mmsqe Date: Mon, 17 Oct 2022 10:33:08 +0800 Subject: [PATCH 05/31] support flags --- app/app.go | 10 +++-- client/flags.go | 6 +++ integration_tests/configs/replica.jsonnet | 2 + integration_tests/test_query_only.py | 52 ++++++++++++++++------- 4 files changed, 50 insertions(+), 20 deletions(-) create mode 100644 client/flags.go diff --git a/app/app.go b/app/app.go index 0cbc63cc5a..22df001fa2 100644 --- a/app/app.go +++ b/app/app.go @@ -124,7 +124,7 @@ import ( gravitytypes "github.com/peggyjv/gravity-bridge/module/v2/x/gravity/types" // this line is used by starport scaffolding # stargate/app/moduleImport - + cronosappclient "github.com/crypto-org-chain/cronos/client" cronosfile "github.com/crypto-org-chain/cronos/file" "github.com/crypto-org-chain/cronos/versiondb" "github.com/crypto-org-chain/cronos/versiondb/tmdb" @@ -378,10 +378,12 @@ func New( panic(err) } versionDB := tmdb.NewStore(plainDB, historyDB, changesetDB) - directory := filepath.Join(rootDir, "data", "file_streamer") + // TODO: sync block data for grpc only mode + isLocal := cast.ToBool(appOpts.Get(cronosappclient.FlagIsLocal)) + remoteUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteUrl)) watcher := cronosfile.NewBlockFileWatcher(func(blockNum int64) string { - return cronosfile.GetLocalDataFileName(directory, blockNum) - }, true) + return fmt.Sprintf("%s/%s", remoteUrl, cronosfile.DataFileName(blockNum)) + }, isLocal) watcher.Start(1, time.Microsecond) go func() { // max retry for temporary io error diff --git a/client/flags.go b/client/flags.go new file mode 100644 index 0000000000..e707b64ab6 --- /dev/null +++ b/client/flags.go @@ -0,0 +1,6 @@ +package client + +const FlagStreamers = "streamers" +const FlagIsLocal = "is-local" +const FlagLocalDirectory = "local-directory" +const FlagRemoteUrl = "remote-url" diff --git a/integration_tests/configs/replica.jsonnet b/integration_tests/configs/replica.jsonnet index 45c4fb95f1..0341e1682b 100644 --- a/integration_tests/configs/replica.jsonnet +++ b/integration_tests/configs/replica.jsonnet @@ -6,6 +6,8 @@ config { 'json-rpc': { enable: false, }, + 'is-local': false, + 'remote-url': 'http://localhost:8080', 'grpc-only': true, }, }, diff --git a/integration_tests/test_query_only.py b/integration_tests/test_query_only.py index 797df5ec56..f834f7a398 100644 --- a/integration_tests/test_query_only.py +++ b/integration_tests/test_query_only.py @@ -1,31 +1,51 @@ +from functools import partial +from http.server import HTTPServer, SimpleHTTPRequestHandler from pathlib import Path +from threading import Thread +from typing import NamedTuple import pytest -from .network import setup_custom_cronos -from .utils import ( - ADDRS, -) +from .network import Cronos, setup_custom_cronos +from .utils import ADDRS, wait_for_port -@pytest.fixture(scope="module") -def primary(tmp_path_factory): - path = tmp_path_factory.mktemp("cronos-primary") - yield from setup_custom_cronos( - path, 26750, Path(__file__).parent / "configs/primary.jsonnet" - ) +class Network(NamedTuple): + primary: Cronos + replica: Cronos @pytest.fixture(scope="module") -def replica(tmp_path_factory): +def network(request, tmp_path_factory): + base = Path(__file__).parent / "configs" + # primary + path = tmp_path_factory.mktemp("cronos-primary") + gen = setup_custom_cronos( + path, 26750, base / "primary.jsonnet" + ) + primary = next(gen) + print("primary path:", path) + # http server + dir = path / "cronos_777-1/node0/data/file_streamer" + print("dir: ", dir) + port = 8080 + handler = partial(SimpleHTTPRequestHandler, directory=dir) + httpd = HTTPServer(("localhost", port), handler) + thread = Thread(target=httpd.serve_forever) + thread.setDaemon(True) + thread.start() + wait_for_port(port) + # replica path = tmp_path_factory.mktemp("cronos-replica") - yield from setup_custom_cronos( - path, 26770, Path(__file__).parent / "configs/replica.jsonnet" + gen = setup_custom_cronos( + path, 26770, base / "replica.jsonnet" ) + replica = next(gen) + yield Network(primary, replica) -def test_basic(primary, replica): - pw3 = primary.w3 - rw3 = replica.w3 +def test_basic(network): + pw3 = network.primary.w3 + rw3 = network.replica.w3 validator = ADDRS["validator"] community = ADDRS["community"] def print_balance(): From a0f319fd8ef25fcc7c811d951a83d367f9e88d6b Mon Sep 17 00:00:00 2001 From: mmsqe Date: Wed, 19 Oct 2022 09:10:42 +0800 Subject: [PATCH 06/31] sync from last version --- app/app.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/app/app.go b/app/app.go index 22df001fa2..3b404fcbe1 100644 --- a/app/app.go +++ b/app/app.go @@ -378,13 +378,22 @@ func New( panic(err) } versionDB := tmdb.NewStore(plainDB, historyDB, changesetDB) + latestVersion, err := versionDB.GetLatestVersion() + if err != nil { + panic(err) + } + startBlockNum := latestVersion + 1 + if startBlockNum <= 0 { + startBlockNum = 1 // min 1 + } + fmt.Printf("mm-startBlockNum: %+v, %+v\n", startBlockNum, err) // TODO: sync block data for grpc only mode isLocal := cast.ToBool(appOpts.Get(cronosappclient.FlagIsLocal)) remoteUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteUrl)) watcher := cronosfile.NewBlockFileWatcher(func(blockNum int64) string { return fmt.Sprintf("%s/%s", remoteUrl, cronosfile.DataFileName(blockNum)) }, isLocal) - watcher.Start(1, time.Microsecond) + watcher.Start(startBlockNum, time.Microsecond) go func() { // max retry for temporary io error const maxRetry = 3 From 0418958cc0d211de3087d3c8e44d9c179cf1c6fe Mon Sep 17 00:00:00 2001 From: mmsqe Date: Thu, 20 Oct 2022 09:24:56 +0800 Subject: [PATCH 07/31] mute log --- integration_tests/test_query_only.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/integration_tests/test_query_only.py b/integration_tests/test_query_only.py index f834f7a398..6bc3a2a5b7 100644 --- a/integration_tests/test_query_only.py +++ b/integration_tests/test_query_only.py @@ -9,13 +9,21 @@ from .network import Cronos, setup_custom_cronos from .utils import ADDRS, wait_for_port + class Network(NamedTuple): primary: Cronos replica: Cronos +class QuietServer(SimpleHTTPRequestHandler): + def __init__(self, dir: str, *args, **kwargs): + super().__init__(directory=dir, *args, **kwargs) + + def log_message(self, format, *args): + pass + @pytest.fixture(scope="module") -def network(request, tmp_path_factory): +def network(tmp_path_factory): base = Path(__file__).parent / "configs" # primary path = tmp_path_factory.mktemp("cronos-primary") @@ -28,8 +36,7 @@ def network(request, tmp_path_factory): dir = path / "cronos_777-1/node0/data/file_streamer" print("dir: ", dir) port = 8080 - handler = partial(SimpleHTTPRequestHandler, directory=dir) - httpd = HTTPServer(("localhost", port), handler) + httpd = HTTPServer(("localhost", port), partial(QuietServer, dir)) thread = Thread(target=httpd.serve_forever) thread.setDaemon(True) thread.start() @@ -50,7 +57,7 @@ def test_basic(network): community = ADDRS["community"] def print_balance(): print(pw3.eth.get_balance(community)) - # print(rw3.eth.get_balance(community)) + print(rw3.eth.get_balance(community)) print_balance() txhash = pw3.eth.send_transaction( From 28d3e06b212cf062b1b9b39f2013e24c455e9a5c Mon Sep 17 00:00:00 2001 From: mmsqe Date: Thu, 20 Oct 2022 09:25:25 +0800 Subject: [PATCH 08/31] grpc only --- app/app.go | 64 +++++++++++++++------------- client/flags.go | 1 + integration_tests/test_query_only.py | 2 +- 3 files changed, 36 insertions(+), 31 deletions(-) diff --git a/app/app.go b/app/app.go index 3b404fcbe1..0bd8a48728 100644 --- a/app/app.go +++ b/app/app.go @@ -386,38 +386,42 @@ func New( if startBlockNum <= 0 { startBlockNum = 1 // min 1 } - fmt.Printf("mm-startBlockNum: %+v, %+v\n", startBlockNum, err) - // TODO: sync block data for grpc only mode - isLocal := cast.ToBool(appOpts.Get(cronosappclient.FlagIsLocal)) - remoteUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteUrl)) - watcher := cronosfile.NewBlockFileWatcher(func(blockNum int64) string { - return fmt.Sprintf("%s/%s", remoteUrl, cronosfile.DataFileName(blockNum)) - }, isLocal) - watcher.Start(startBlockNum, time.Microsecond) - go func() { - // max retry for temporary io error - const maxRetry = 3 - retry := 0 - chData, chErr := watcher.SubscribeData(), watcher.SubscribeError() - for { - select { - case data := <-chData: - pairs, err := cronosfile.DecodeData(data.Data) - if err != nil { - panic(err) - } - retry = 0 - fmt.Printf("mm-pairs: %+v\n", pairs) - versionDB.PutAtVersion(data.BlockNum, pairs) - case err := <-chErr: - retry++ - if retry == maxRetry { - // data corrupt - panic(err) + directory := filepath.Join(rootDir, "data", "file_streamer") + isGrpcOnly := cast.ToBool(appOpts.Get(cronosappclient.FlagIsGrpcOnly)) + fmt.Printf("mm-startBlockNum: %+v, %+v, %+v, %+v\n", startBlockNum, directory, isGrpcOnly, err) + if isGrpcOnly { + // TODO: sync block data for grpc only mode + isLocal := cast.ToBool(appOpts.Get(cronosappclient.FlagIsLocal)) + remoteUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteUrl)) + watcher := cronosfile.NewBlockFileWatcher(func(blockNum int64) string { + return fmt.Sprintf("%s/%s", remoteUrl, cronosfile.DataFileName(blockNum)) + }, isLocal) + watcher.Start(startBlockNum, time.Microsecond) + go func() { + // max retry for temporary io error + const maxRetry = 3 + retry := 0 + chData, chErr := watcher.SubscribeData(), watcher.SubscribeError() + for { + select { + case data := <-chData: + pairs, err := cronosfile.DecodeData(data.Data) + if err != nil { + panic(err) + } + retry = 0 + fmt.Printf("mm-pairs: %+v\n", len(pairs)) + versionDB.PutAtVersion(data.BlockNum, pairs) + case err := <-chErr: + retry++ + if retry == maxRetry { + // data corrupt + panic(err) + } } } - } - }() + }() + } // default to exposing all exposeStoreKeys := make([]storetypes.StoreKey, 0, len(keys)) diff --git a/client/flags.go b/client/flags.go index e707b64ab6..815fc043b2 100644 --- a/client/flags.go +++ b/client/flags.go @@ -1,6 +1,7 @@ package client const FlagStreamers = "streamers" +const FlagIsGrpcOnly = "grpc-only" const FlagIsLocal = "is-local" const FlagLocalDirectory = "local-directory" const FlagRemoteUrl = "remote-url" diff --git a/integration_tests/test_query_only.py b/integration_tests/test_query_only.py index 6bc3a2a5b7..f04599f5ec 100644 --- a/integration_tests/test_query_only.py +++ b/integration_tests/test_query_only.py @@ -44,7 +44,7 @@ def network(tmp_path_factory): # replica path = tmp_path_factory.mktemp("cronos-replica") gen = setup_custom_cronos( - path, 26770, base / "replica.jsonnet" + path, 26770, base / "replica.jsonnet", ) replica = next(gen) yield Network(primary, replica) From 1f590ad54e9381675fc583c25d270c91d6690351 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Fri, 21 Oct 2022 09:28:56 +0800 Subject: [PATCH 09/31] concurrent sync --- app/app.go | 28 +++++++-------- file/watcher.go | 95 +++++++++++++++++++++++++++++++++++++------------ 2 files changed, 85 insertions(+), 38 deletions(-) diff --git a/app/app.go b/app/app.go index 0bd8a48728..673973418f 100644 --- a/app/app.go +++ b/app/app.go @@ -382,36 +382,32 @@ func New( if err != nil { panic(err) } - startBlockNum := latestVersion + 1 - if startBlockNum <= 0 { - startBlockNum = 1 // min 1 + startBlockNum := latestVersion + if startBlockNum < 0 { + startBlockNum = 0 } - directory := filepath.Join(rootDir, "data", "file_streamer") isGrpcOnly := cast.ToBool(appOpts.Get(cronosappclient.FlagIsGrpcOnly)) - fmt.Printf("mm-startBlockNum: %+v, %+v, %+v, %+v\n", startBlockNum, directory, isGrpcOnly, err) + fmt.Printf("mm-startBlockNum: %+v, %+v, %+v\n", startBlockNum, isGrpcOnly, err) if isGrpcOnly { - // TODO: sync block data for grpc only mode + directory := filepath.Join(rootDir, "data", "file_streamer") isLocal := cast.ToBool(appOpts.Get(cronosappclient.FlagIsLocal)) remoteUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteUrl)) - watcher := cronosfile.NewBlockFileWatcher(func(blockNum int64) string { + concurrency := 21 + watcher := cronosfile.NewBlockFileWatcher(concurrency, func(blockNum int) string { return fmt.Sprintf("%s/%s", remoteUrl, cronosfile.DataFileName(blockNum)) - }, isLocal) - watcher.Start(startBlockNum, time.Microsecond) + }, isLocal, directory) go func() { // max retry for temporary io error - const maxRetry = 3 + maxRetry := concurrency * 2 retry := 0 chData, chErr := watcher.SubscribeData(), watcher.SubscribeError() for { select { case data := <-chData: - pairs, err := cronosfile.DecodeData(data.Data) - if err != nil { - panic(err) - } + fmt.Printf("mm-file: %s\n", data.FilePath) retry = 0 - fmt.Printf("mm-pairs: %+v\n", len(pairs)) - versionDB.PutAtVersion(data.BlockNum, pairs) + // fmt.Printf("mm-pairs: %+v\n", len(pairs)) + // versionDB.PutAtVersion(int64(data.BlockNum), pairs) case err := <-chErr: retry++ if retry == maxRetry { diff --git a/file/watcher.go b/file/watcher.go index 5bdc0e4fc8..22056bd7ef 100644 --- a/file/watcher.go +++ b/file/watcher.go @@ -32,7 +32,10 @@ func (d *localFileDownloader) GetData(path string) ([]byte, error) { type httpFileDownloader struct{} func (d *httpFileDownloader) GetData(path string) ([]byte, error) { - resp, err := http.Get(path) //nolint + c := &http.Client{ + Timeout: time.Minute, + } + resp, err := c.Get(path) //nolint if err != nil { return nil, err } @@ -49,28 +52,34 @@ func (d *httpFileDownloader) GetData(path string) ([]byte, error) { } type BlockData struct { - BlockNum int64 - Data []byte + BlockNum int + FilePath string } type BlockFileWatcher struct { - getFilePath func(blockNum int64) string + concurrency int + getFilePath func(blockNum int) string downloader fileDownloader chData chan *BlockData chError chan error chDone chan bool startLock *sync.Mutex + directory string } func NewBlockFileWatcher( - getFilePath func(blockNum int64) string, + concurrency int, + getFilePath func(blockNum int) string, isLocal bool, + directory string, ) *BlockFileWatcher { w := &BlockFileWatcher{ + concurrency: concurrency, getFilePath: getFilePath, chData: make(chan *BlockData), chError: make(chan error), startLock: new(sync.Mutex), + directory: directory, } if isLocal { w.downloader = new(localFileDownloader) @@ -80,11 +89,11 @@ func NewBlockFileWatcher( return w } -func DataFileName(blockNum int64) string { +func DataFileName(blockNum int) string { return fmt.Sprintf("block-%d-data", blockNum) } -func GetLocalDataFileName(directory string, blockNum int64) string { +func GetLocalDataFileName(directory string, blockNum int) string { return fmt.Sprintf("%s/%s", directory, DataFileName(blockNum)) } @@ -96,8 +105,36 @@ func (w *BlockFileWatcher) SubscribeError() <-chan error { return w.chError } +func (w *BlockFileWatcher) fetch(blockNum int) (string, error) { + path := w.getFilePath(blockNum) + data, err := w.downloader.GetData(path) + fmt.Printf("mm-path: %+v, %+v, %+v\n", path, len(data), err) + if err != nil { + if err != errNotExist { + // avoid blocked by error when not subscribe + select { + case w.chError <- err: + default: + } + } + return "", err + } + w.chData <- &BlockData{ + BlockNum: blockNum, + FilePath: path, + } + file := GetLocalDataFileName(w.directory, blockNum) + fmt.Printf("mm-file: %+v\n", file) + err = os.WriteFile(file, data, 0644) + if err != nil { + fmt.Printf("mm-writeErr: %+v\n", err) + return "", err + } + return file, nil +} + func (w *BlockFileWatcher) Start( - blockNum int64, + blockNum int, interval time.Duration, ) { w.startLock.Lock() @@ -108,28 +145,42 @@ func (w *BlockFileWatcher) Start( w.chDone = make(chan bool) go func() { + finishedBlockNums := make(map[int]bool) for { select { case <-w.chDone: return default: - path := w.getFilePath(blockNum) - data, err := w.downloader.GetData(path) - if err != nil { - if err != errNotExist { - // avoid blocked by error when not subscribe - select { - case w.chError <- err: - default: - } + wg := new(sync.WaitGroup) + resultFiles := make([]string, w.concurrency) + for i := 1; i <= w.concurrency; i++ { + fmt.Println("mm-start: ", i) + if !finishedBlockNums[blockNum+i] { + wg.Add(1) + go func(i int) { + f, err := w.fetch(blockNum + i) + if err == nil { + resultFiles[i-1] = f + } + wg.Done() + }(i) } - } else { - w.chData <- &BlockData{ - BlockNum: blockNum, - Data: data, + } + wg.Wait() + errReached := false + finishedBlockNums = make(map[int]bool) + currentBlockNum := blockNum + for i, f := range resultFiles { + b := currentBlockNum + i + 1 + if f == "" { + errReached = true + } else { + finishedBlockNums[b] = true + if !errReached { + blockNum = b + } } - blockNum++ } time.Sleep(interval) } From 9531101953de03565b27ccfe364f4921f1e7a3fd Mon Sep 17 00:00:00 2001 From: mmsqe Date: Fri, 21 Oct 2022 09:29:02 +0800 Subject: [PATCH 10/31] configurable concurrency --- app/app.go | 3 +-- client/flags.go | 1 + go.mod | 10 ++++----- go.sum | 26 +++++++++++++---------- gomod2nix.toml | 20 ++++++++--------- integration_tests/configs/replica.jsonnet | 1 + 6 files changed, 33 insertions(+), 28 deletions(-) diff --git a/app/app.go b/app/app.go index 673973418f..9bee8fd7e0 100644 --- a/app/app.go +++ b/app/app.go @@ -6,7 +6,6 @@ import ( "net/http" "os" "path/filepath" - "time" "github.com/crypto-org-chain/cronos/x/cronos" "github.com/crypto-org-chain/cronos/x/cronos/middleware" @@ -392,7 +391,7 @@ func New( directory := filepath.Join(rootDir, "data", "file_streamer") isLocal := cast.ToBool(appOpts.Get(cronosappclient.FlagIsLocal)) remoteUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteUrl)) - concurrency := 21 + concurrency := cast.ToInt(appOpts.Get(cronosappclient.FlagConcurrency)) watcher := cronosfile.NewBlockFileWatcher(concurrency, func(blockNum int) string { return fmt.Sprintf("%s/%s", remoteUrl, cronosfile.DataFileName(blockNum)) }, isLocal, directory) diff --git a/client/flags.go b/client/flags.go index 815fc043b2..48d00913de 100644 --- a/client/flags.go +++ b/client/flags.go @@ -5,3 +5,4 @@ const FlagIsGrpcOnly = "grpc-only" const FlagIsLocal = "is-local" const FlagLocalDirectory = "local-directory" const FlagRemoteUrl = "remote-url" +const FlagConcurrency = "concurrency" diff --git a/go.mod b/go.mod index 3048a6be54..78ff2d7709 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/cosmos/gogoproto v1.4.3 github.com/cosmos/ibc-go/v5 v5.0.0 github.com/ethereum/go-ethereum v1.10.19 - github.com/evmos/ethermint v0.6.1-0.20221003153722-491c3da7ebd7 + github.com/evmos/ethermint v0.6.1-0.20221024154336-dca6d45cd75e github.com/gogo/protobuf v1.3.3 github.com/golang/protobuf v1.5.2 github.com/gorilla/mux v1.8.0 @@ -20,10 +20,10 @@ require ( github.com/spf13/cast v1.5.0 github.com/spf13/cobra v1.6.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.1 github.com/tendermint/tendermint v0.34.24 github.com/tendermint/tm-db v0.6.7 - google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a + google.golang.org/genproto v0.0.0-20221018160656-63c7b68cfc55 google.golang.org/grpc v1.50.1 google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8 gopkg.in/yaml.v2 v2.4.0 @@ -158,7 +158,7 @@ require ( github.com/rs/zerolog v1.27.0 // indirect github.com/sasha-s/go-deadlock v0.3.1 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect - github.com/spf13/afero v1.8.2 // indirect + github.com/spf13/afero v1.9.2 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.13.0 // indirect github.com/status-im/keycard-go v0.0.0-20200402102358-957c09536969 // indirect @@ -182,7 +182,7 @@ require ( golang.org/x/sys v0.1.0 // indirect golang.org/x/term v0.1.0 // indirect golang.org/x/text v0.4.0 // indirect - golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.93.0 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 3be31013c8..53237e3370 100644 --- a/go.sum +++ b/go.sum @@ -336,8 +336,8 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go. github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/go-ethereum v1.10.19 h1:EOR5JbL4MD5yeOqv8W2iC1s4NximrTjqFccUz8lyBRA= github.com/ethereum/go-ethereum v1.10.19/go.mod h1:IJBNMtzKcNHPtllYihy6BL2IgK1u+32JriaTbdt4v+w= -github.com/evmos/ethermint v0.6.1-0.20221003153722-491c3da7ebd7 h1:TeJivwA5zRWPni7ipvPfn7aiYfNxPLT/xX+M6jBpbHo= -github.com/evmos/ethermint v0.6.1-0.20221003153722-491c3da7ebd7/go.mod h1:a7+BifJFvJqK7b4MJ2WKqIs9LSPBU1ojbm3okFQYEfU= +github.com/evmos/ethermint v0.6.1-0.20221024154336-dca6d45cd75e h1:2ahOWcXLrVeJsIoBBMj0P/zcvZCK/9QgrR2aJop1rYo= +github.com/evmos/ethermint v0.6.1-0.20221024154336-dca6d45cd75e/go.mod h1:k8w7pACqdY6g9nl/qSguOr1pjfo9JO5g6RIxHaNtLZ8= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= @@ -388,6 +388,7 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -811,12 +812,12 @@ github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo/v2 v2.2.0 h1:3ZNA3L1c5FYDFTTxbFeVGGD8jYvjYauHD30YgLxVsNI= +github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.20.2 h1:8uQq0zMgLEfa0vRrrBgaJF2gyW9Da9BmfGV+OyUzfkY= +github.com/onsi/gomega v1.22.1 h1:pY8O4lBfsHKZHM/6nrxkhVPUznOlIu3quZcKP/M20KI= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= @@ -952,8 +953,8 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= -github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= +github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw= +github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= @@ -979,8 +980,9 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -989,8 +991,9 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= @@ -1449,8 +1452,9 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= @@ -1597,8 +1601,8 @@ google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljW google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a h1:GH6UPn3ixhWcKDhpnEC55S75cerLPdpp3hrhfKYjZgw= -google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221018160656-63c7b68cfc55 h1:U1u4KB2kx6KR/aJDjQ97hZ15wQs8ZPvDcGcRynBhkvg= +google.golang.org/genproto v0.0.0-20221018160656-63c7b68cfc55/go.mod h1:45EK0dUbEZ2NHjCeAd2LXmyjAgGUGrpGROgjhC3ADck= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= diff --git a/gomod2nix.toml b/gomod2nix.toml index 408f0be0d3..7f787a5d2c 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -175,8 +175,8 @@ schema = 3 hash = "sha256-7FPnTGcCb8Xd1QVR+6PmGTaHdTY1mm/8osFTW1JLuG8=" replaced = "github.com/ethereum/go-ethereum" [mod."github.com/evmos/ethermint"] - version = "v0.6.1-0.20221003153722-491c3da7ebd7" - hash = "sha256-vnfjk57gYa+F8nn0LByX/B1LV8PY2Jvm8vXV6be4ufc=" + version = "v0.6.1-0.20221024154336-dca6d45cd75e" + hash = "sha256-pDhVKxRe7KKo2/CnyfBMOO4zpy/sPt5fmgU2G4ax5kw=" [mod."github.com/felixge/httpsnoop"] version = "v1.0.2" hash = "sha256-hj6FZQ1fDAV+1wGIViAt8XaAkWZ1I5vJzgjIJa7XRBA=" @@ -432,8 +432,8 @@ schema = 3 version = "v3.21.4-0.20210419000835-c7a38de76ee5+incompatible" hash = "sha256-oqIqyFquWabIE6DID6uTEc8oFEmM1rVu2ATn3toiCEg=" [mod."github.com/spf13/afero"] - version = "v1.8.2" - hash = "sha256-ERuG4GK1LOn72DRbT9URpo1QoVIm2A3mPtlf+xSPgx8=" + version = "v1.9.2" + hash = "sha256-R1mir7Fu95QK+YL99U14RGbLJzxqWRH5rSFpssgJvzA=" [mod."github.com/spf13/cast"] version = "v1.5.0" hash = "sha256-Pdp+wC5FWqyJKzyYHb7JCcV9BoJk/sxQw6nLyuLJvuQ=" @@ -453,8 +453,8 @@ schema = 3 version = "v0.0.0-20200402102358-957c09536969" hash = "sha256-yddXXuu6mEFEO2/K6c1tWymeBKzOcvLQnNsFGRjtfXk=" [mod."github.com/stretchr/testify"] - version = "v1.8.0" - hash = "sha256-LDxBAebK+A06y4vbH7cd1sVBOameIY81Xm8/9OPZh7o=" + version = "v1.8.1" + hash = "sha256-3e0vOJLgCMAan+GfaGN8RGZdarh5iCavM6flf6YMNPk=" [mod."github.com/subosito/gotenv"] version = "v1.4.1" hash = "sha256-aHAUs8Gh+BREUulljWr7vBZQg1BLu/M/8UfNK8CWSn8=" @@ -523,8 +523,8 @@ schema = 3 version = "v0.4.0" hash = "sha256-JvyMygdmTvWg7xhbnUB9MMk6WcYXJt8DAj4DYl82Pys=" [mod."golang.org/x/xerrors"] - version = "v0.0.0-20220609144429-65e65417b02f" - hash = "sha256-tl8pv3oddbz2+KoIp7PFDKsxjQF8ocjPF8XPsY3sw38=" + version = "v0.0.0-20220907171357-04be3eba64a2" + hash = "sha256-6+zueutgefIYmgXinOflz8qGDDDj0Zhv+2OkGhBTKno=" [mod."google.golang.org/api"] version = "v0.93.0" hash = "sha256-W17B79osAhObMbrCHlgywEPg9yIPx0ZISDhp9JwFE5A=" @@ -532,8 +532,8 @@ schema = 3 version = "v1.6.7" hash = "sha256-zIxGRHiq4QBvRqkrhMGMGCaVL4iM4TtlYpAi/hrivS4=" [mod."google.golang.org/genproto"] - version = "v0.0.0-20221014213838-99cd37c6964a" - hash = "sha256-zSdk2kbcfWaaJfHxLULI9v38lvEaJb8koC5c59aVZZI=" + version = "v0.0.0-20221018160656-63c7b68cfc55" + hash = "sha256-Z4ynOw+3NkSkWVOrNaeuQC3G9Q6+joNbTLVoF2HegrU=" [mod."google.golang.org/grpc"] version = "v1.50.1" hash = "sha256-38nk4qIme+fE57SsCqNxtCZnc8fyzzi4Sb60uDTT2KE=" diff --git a/integration_tests/configs/replica.jsonnet b/integration_tests/configs/replica.jsonnet index 0341e1682b..b539fd8fab 100644 --- a/integration_tests/configs/replica.jsonnet +++ b/integration_tests/configs/replica.jsonnet @@ -9,6 +9,7 @@ config { 'is-local': false, 'remote-url': 'http://localhost:8080', 'grpc-only': true, + 'concurrency': 4, }, }, } From 649a852404ed66dfa48f1d222d9f4e1fc9232b0a Mon Sep 17 00:00:00 2001 From: mmsqe Date: Fri, 21 Oct 2022 09:29:07 +0800 Subject: [PATCH 11/31] separate synchronizer with streamer instead of sync directly to db --- app/app.go | 54 +++++++++++++++++++++++++++++++++++++------------ file/watcher.go | 40 +++++++++++++++--------------------- 2 files changed, 57 insertions(+), 37 deletions(-) diff --git a/app/app.go b/app/app.go index 9bee8fd7e0..b94473021c 100644 --- a/app/app.go +++ b/app/app.go @@ -6,6 +6,7 @@ import ( "net/http" "os" "path/filepath" + "time" "github.com/crypto-org-chain/cronos/x/cronos" "github.com/crypto-org-chain/cronos/x/cronos/middleware" @@ -377,33 +378,38 @@ func New( panic(err) } versionDB := tmdb.NewStore(plainDB, historyDB, changesetDB) - latestVersion, err := versionDB.GetLatestVersion() - if err != nil { - panic(err) - } - startBlockNum := latestVersion - if startBlockNum < 0 { - startBlockNum = 0 - } isGrpcOnly := cast.ToBool(appOpts.Get(cronosappclient.FlagIsGrpcOnly)) - fmt.Printf("mm-startBlockNum: %+v, %+v, %+v\n", startBlockNum, isGrpcOnly, err) if isGrpcOnly { + latestVersion, err := versionDB.GetLatestVersion() + fmt.Printf("mm-latestVersion: %+v\n", latestVersion) + if err != nil { + panic(err) + } + startBlockNum := latestVersion + if startBlockNum < 0 { + startBlockNum = 0 + } directory := filepath.Join(rootDir, "data", "file_streamer") isLocal := cast.ToBool(appOpts.Get(cronosappclient.FlagIsLocal)) remoteUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteUrl)) concurrency := cast.ToInt(appOpts.Get(cronosappclient.FlagConcurrency)) - watcher := cronosfile.NewBlockFileWatcher(concurrency, func(blockNum int) string { + synchronizer := cronosfile.NewBlockFileWatcher(concurrency, func(blockNum int) string { return fmt.Sprintf("%s/%s", remoteUrl, cronosfile.DataFileName(blockNum)) - }, isLocal, directory) + }, isLocal) + synchronizer.Start(int(startBlockNum), time.Microsecond) go func() { // max retry for temporary io error maxRetry := concurrency * 2 retry := 0 - chData, chErr := watcher.SubscribeData(), watcher.SubscribeError() + chData, chErr := synchronizer.SubscribeData(), synchronizer.SubscribeError() for { select { case data := <-chData: - fmt.Printf("mm-file: %s\n", data.FilePath) + file := cronosfile.GetLocalDataFileName(directory, data.BlockNum) + fmt.Printf("mm-file: %+v\n", file) + if err := os.WriteFile(file, data.Data, 0644); err != nil { + panic(err) + } retry = 0 // fmt.Printf("mm-pairs: %+v\n", len(pairs)) // versionDB.PutAtVersion(int64(data.BlockNum), pairs) @@ -416,6 +422,28 @@ func New( } } }() + + // streamer write the file blk by blk with concurrency 1 + streamer := cronosfile.NewBlockFileWatcher(1, func(blockNum int) string { + return cronosfile.GetLocalDataFileName(directory, blockNum) + }, true) + streamer.Start(int(startBlockNum), time.Microsecond) + go func() { + chData, chErr := streamer.SubscribeData(), streamer.SubscribeError() + for { + select { + case data := <-chData: + pairs, err := cronosfile.DecodeData(data.Data) + fmt.Printf("mm-pairs: %+v, %+v\n", len(pairs), err) + if err == nil { + versionDB.PutAtVersion(int64(data.BlockNum), pairs) + } + case err := <-chErr: + // fail read + panic(err) + } + } + }() } // default to exposing all diff --git a/file/watcher.go b/file/watcher.go index 22056bd7ef..389adf9b29 100644 --- a/file/watcher.go +++ b/file/watcher.go @@ -53,7 +53,7 @@ func (d *httpFileDownloader) GetData(path string) ([]byte, error) { type BlockData struct { BlockNum int - FilePath string + Data []byte } type BlockFileWatcher struct { @@ -64,14 +64,12 @@ type BlockFileWatcher struct { chError chan error chDone chan bool startLock *sync.Mutex - directory string } func NewBlockFileWatcher( concurrency int, getFilePath func(blockNum int) string, isLocal bool, - directory string, ) *BlockFileWatcher { w := &BlockFileWatcher{ concurrency: concurrency, @@ -79,7 +77,6 @@ func NewBlockFileWatcher( chData: make(chan *BlockData), chError: make(chan error), startLock: new(sync.Mutex), - directory: directory, } if isLocal { w.downloader = new(localFileDownloader) @@ -105,10 +102,10 @@ func (w *BlockFileWatcher) SubscribeError() <-chan error { return w.chError } -func (w *BlockFileWatcher) fetch(blockNum int) (string, error) { +func (w *BlockFileWatcher) fetch(blockNum int) error { path := w.getFilePath(blockNum) data, err := w.downloader.GetData(path) - fmt.Printf("mm-path: %+v, %+v, %+v\n", path, len(data), err) + fmt.Printf("mm-fetch: %+v, %+v, %+v\n", blockNum, len(data), err) if err != nil { if err != errNotExist { // avoid blocked by error when not subscribe @@ -117,20 +114,13 @@ func (w *BlockFileWatcher) fetch(blockNum int) (string, error) { default: } } - return "", err + return err } w.chData <- &BlockData{ BlockNum: blockNum, - FilePath: path, + Data: data, } - file := GetLocalDataFileName(w.directory, blockNum) - fmt.Printf("mm-file: %+v\n", file) - err = os.WriteFile(file, data, 0644) - if err != nil { - fmt.Printf("mm-writeErr: %+v\n", err) - return "", err - } - return file, nil + return nil } func (w *BlockFileWatcher) Start( @@ -153,27 +143,24 @@ func (w *BlockFileWatcher) Start( default: wg := new(sync.WaitGroup) - resultFiles := make([]string, w.concurrency) + resultErrs := make([]error, w.concurrency) for i := 1; i <= w.concurrency; i++ { fmt.Println("mm-start: ", i) if !finishedBlockNums[blockNum+i] { wg.Add(1) go func(i int) { - f, err := w.fetch(blockNum + i) - if err == nil { - resultFiles[i-1] = f - } + err := w.fetch(blockNum + i) + resultErrs[i-1] = err wg.Done() }(i) } } wg.Wait() errReached := false - finishedBlockNums = make(map[int]bool) currentBlockNum := blockNum - for i, f := range resultFiles { + for i, err := range resultErrs { b := currentBlockNum + i + 1 - if f == "" { + if err != nil { errReached = true } else { finishedBlockNums[b] = true @@ -182,6 +169,11 @@ func (w *BlockFileWatcher) Start( } } } + for k := range finishedBlockNums { + if k <= blockNum { + delete(finishedBlockNums, k) + } + } time.Sleep(interval) } } From e10a999e42925014eb352ec7d9571501d77ac3f0 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Fri, 21 Oct 2022 09:29:11 +0800 Subject: [PATCH 12/31] bigger interval --- app/app.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/app.go b/app/app.go index b94473021c..0d62e4409a 100644 --- a/app/app.go +++ b/app/app.go @@ -393,10 +393,11 @@ func New( isLocal := cast.ToBool(appOpts.Get(cronosappclient.FlagIsLocal)) remoteUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteUrl)) concurrency := cast.ToInt(appOpts.Get(cronosappclient.FlagConcurrency)) + interval := time.Second synchronizer := cronosfile.NewBlockFileWatcher(concurrency, func(blockNum int) string { return fmt.Sprintf("%s/%s", remoteUrl, cronosfile.DataFileName(blockNum)) }, isLocal) - synchronizer.Start(int(startBlockNum), time.Microsecond) + synchronizer.Start(int(startBlockNum), interval) go func() { // max retry for temporary io error maxRetry := concurrency * 2 @@ -427,7 +428,7 @@ func New( streamer := cronosfile.NewBlockFileWatcher(1, func(blockNum int) string { return cronosfile.GetLocalDataFileName(directory, blockNum) }, true) - streamer.Start(int(startBlockNum), time.Microsecond) + streamer.Start(int(startBlockNum), interval) go func() { chData, chErr := streamer.SubscribeData(), streamer.SubscribeError() for { From cc22b7bcf1869dcf84bff9741fd4d71cf7880d5d Mon Sep 17 00:00:00 2001 From: mmsqe Date: Fri, 21 Oct 2022 09:29:16 +0800 Subject: [PATCH 13/31] fix next blk with test --- app/app.go | 5 +++-- file/watcher.go | 21 +++++++++++---------- file/watcher_test.go | 27 ++++++++++++++------------- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/app/app.go b/app/app.go index 0d62e4409a..6e80c128a6 100644 --- a/app/app.go +++ b/app/app.go @@ -389,6 +389,7 @@ func New( if startBlockNum < 0 { startBlockNum = 0 } + nextBlockNum := int(startBlockNum) + 1 directory := filepath.Join(rootDir, "data", "file_streamer") isLocal := cast.ToBool(appOpts.Get(cronosappclient.FlagIsLocal)) remoteUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteUrl)) @@ -397,7 +398,7 @@ func New( synchronizer := cronosfile.NewBlockFileWatcher(concurrency, func(blockNum int) string { return fmt.Sprintf("%s/%s", remoteUrl, cronosfile.DataFileName(blockNum)) }, isLocal) - synchronizer.Start(int(startBlockNum), interval) + synchronizer.Start(nextBlockNum, interval) go func() { // max retry for temporary io error maxRetry := concurrency * 2 @@ -428,7 +429,7 @@ func New( streamer := cronosfile.NewBlockFileWatcher(1, func(blockNum int) string { return cronosfile.GetLocalDataFileName(directory, blockNum) }, true) - streamer.Start(int(startBlockNum), interval) + streamer.Start(nextBlockNum, interval) go func() { chData, chErr := streamer.SubscribeData(), streamer.SubscribeError() for { diff --git a/file/watcher.go b/file/watcher.go index 389adf9b29..3eceb12b90 100644 --- a/file/watcher.go +++ b/file/watcher.go @@ -104,6 +104,7 @@ func (w *BlockFileWatcher) SubscribeError() <-chan error { func (w *BlockFileWatcher) fetch(blockNum int) error { path := w.getFilePath(blockNum) + // TBC: skip if exist path to avoid dup download data, err := w.downloader.GetData(path) fmt.Printf("mm-fetch: %+v, %+v, %+v\n", blockNum, len(data), err) if err != nil { @@ -132,7 +133,6 @@ func (w *BlockFileWatcher) Start( if w.chDone != nil { return } - w.chDone = make(chan bool) go func() { finishedBlockNums := make(map[int]bool) @@ -144,28 +144,29 @@ func (w *BlockFileWatcher) Start( default: wg := new(sync.WaitGroup) resultErrs := make([]error, w.concurrency) - for i := 1; i <= w.concurrency; i++ { - fmt.Println("mm-start: ", i) - if !finishedBlockNums[blockNum+i] { + for i := 0; i < w.concurrency; i++ { + nextBlockNum := blockNum + i + fmt.Println("mm-start: ", nextBlockNum) + if !finishedBlockNums[nextBlockNum] { wg.Add(1) - go func(i int) { - err := w.fetch(blockNum + i) - resultErrs[i-1] = err + go func(nextBlockNum, i int) { + err := w.fetch(nextBlockNum) + resultErrs[i] = err wg.Done() - }(i) + }(nextBlockNum, i) } } wg.Wait() errReached := false currentBlockNum := blockNum for i, err := range resultErrs { - b := currentBlockNum + i + 1 + b := currentBlockNum + i if err != nil { errReached = true } else { finishedBlockNums[b] = true if !errReached { - blockNum = b + blockNum = b + 1 } } } diff --git a/file/watcher_test.go b/file/watcher_test.go index a5f4b2902c..eee2204f40 100644 --- a/file/watcher_test.go +++ b/file/watcher_test.go @@ -21,21 +21,22 @@ func setupDirectory(t *testing.T, directory string) func(t *testing.T) { } } -func setupBlockFiles(directory string, start, end int64) { +func setupBlockFiles(directory string, start, end int) { for i := start; i <= end; i++ { file := GetLocalDataFileName(directory, i) os.WriteFile(file, []byte(fmt.Sprint("block", i)), 0644) } } -func start(watcher *BlockFileWatcher, endBlockNum int64) int64 { - watcher.Start(1, time.Microsecond) - counter := int64(0) +func start(watcher *BlockFileWatcher, startBlockNum, endBlockNum int) int { + watcher.Start(startBlockNum, time.Microsecond) + counter := 0 for data := range watcher.SubscribeData() { if data != nil && len(data.Data) > 0 { counter++ } if data.BlockNum == endBlockNum { + watcher.Close() return counter } } @@ -45,19 +46,19 @@ func start(watcher *BlockFileWatcher, endBlockNum int64) int64 { func TestFileWatcher(t *testing.T) { directory := "tmp" teardown := setupDirectory(t, directory) - startBlockNum := int64(1) - endBlockNum := int64(2) - + startBlockNum := 1 + endBlockNum := 2 + concurrency := 1 defer teardown(t) t.Run("when sync via local", func(t *testing.T) { setupBlockFiles(directory, startBlockNum, endBlockNum) - watcher := NewBlockFileWatcher(func(blockNum int64) string { + watcher := NewBlockFileWatcher(concurrency, func(blockNum int) string { return GetLocalDataFileName(directory, blockNum) }, true) - total := start(watcher, endBlockNum) + total := start(watcher, startBlockNum, endBlockNum) expected := endBlockNum - startBlockNum + 1 - require.Equal(t, total, expected) + require.Equal(t, expected, total) }) t.Run("when sync via http", func(t *testing.T) { @@ -68,11 +69,11 @@ func TestFileWatcher(t *testing.T) { go func() { log.Fatal(http.ListenAndServe(":"+port, nil)) }() - watcher := NewBlockFileWatcher(func(blockNum int64) string { + watcher := NewBlockFileWatcher(concurrency, func(blockNum int) string { return fmt.Sprintf("http://localhost:%s/%s", port, DataFileName(blockNum)) }, false) - total := start(watcher, endBlockNum) + total := start(watcher, startBlockNum, endBlockNum) expected := endBlockNum - startBlockNum + 1 - require.Equal(t, total, expected) + require.Equal(t, expected, total) }) } From fe8361ef671cc968f2d867680d2348cfc7461289 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Fri, 21 Oct 2022 17:21:27 +0800 Subject: [PATCH 14/31] fix test --- integration_tests/configs/replica.jsonnet | 2 +- integration_tests/test_query_only.py | 94 ++++++++++++++++++----- 2 files changed, 74 insertions(+), 22 deletions(-) diff --git a/integration_tests/configs/replica.jsonnet b/integration_tests/configs/replica.jsonnet index b539fd8fab..1d55df2020 100644 --- a/integration_tests/configs/replica.jsonnet +++ b/integration_tests/configs/replica.jsonnet @@ -9,7 +9,7 @@ config { 'is-local': false, 'remote-url': 'http://localhost:8080', 'grpc-only': true, - 'concurrency': 4, + 'concurrency': 1, }, }, } diff --git a/integration_tests/test_query_only.py b/integration_tests/test_query_only.py index f04599f5ec..b00399799f 100644 --- a/integration_tests/test_query_only.py +++ b/integration_tests/test_query_only.py @@ -1,3 +1,7 @@ +import os +import signal +import subprocess +import time from functools import partial from http.server import HTTPServer, SimpleHTTPRequestHandler from pathlib import Path @@ -5,8 +9,10 @@ from typing import NamedTuple import pytest +import requests +from pystarport import ports -from .network import Cronos, setup_custom_cronos +from .network import Cronos from .utils import ADDRS, wait_for_port @@ -22,18 +28,38 @@ def __init__(self, dir: str, *args, **kwargs): def log_message(self, format, *args): pass + +def exec(config, path, base_port): + cmd = [ + "pystarport", + "init", + "--config", + config, + "--data", + path, + "--base_port", + str(base_port), + "--no_remove", + ] + print(*cmd) + subprocess.run(cmd, check=True) + return subprocess.Popen( + ["pystarport", "start", "--data", path, "--quiet"], + preexec_fn=os.setsid, + ) + + @pytest.fixture(scope="module") def network(tmp_path_factory): + chain_id = "cronos_777-1" base = Path(__file__).parent / "configs" # primary - path = tmp_path_factory.mktemp("cronos-primary") - gen = setup_custom_cronos( - path, 26750, base / "primary.jsonnet" - ) - primary = next(gen) - print("primary path:", path) + path0 = tmp_path_factory.mktemp("cronos-primary") + base_port0 = 26750 + procs = [exec(base / "primary.jsonnet", path0, base_port0)] + # http server - dir = path / "cronos_777-1/node0/data/file_streamer" + dir = path0 / f"{chain_id}/node0/data/file_streamer" print("dir: ", dir) port = 8080 httpd = HTTPServer(("localhost", port), partial(QuietServer, dir)) @@ -41,32 +67,58 @@ def network(tmp_path_factory): thread.setDaemon(True) thread.start() wait_for_port(port) + # replica - path = tmp_path_factory.mktemp("cronos-replica") - gen = setup_custom_cronos( - path, 26770, base / "replica.jsonnet", - ) - replica = next(gen) - yield Network(primary, replica) + path1 = tmp_path_factory.mktemp("cronos-replica") + base_port1 = 26770 + procs.append(exec(base / "replica.jsonnet", path1, base_port1)) + try: + wait_for_port(ports.evmrpc_port(base_port0)) + wait_for_port(ports.grpc_port(base_port1)) + yield Network(Cronos(path0 / chain_id), Cronos(path1 / chain_id)) + finally: + for proc in procs: + print("killing:", proc.pid) + os.killpg(os.getpgid(proc.pid), signal.SIGTERM) + proc.wait() + print("killed:", proc.pid) + + +def grpc_call(p, address): + url = f"http://127.0.0.1:{p}/cosmos/bank/v1beta1/balances/{address}" + response = requests.get(url) + if not response.ok: + raise Exception( + f"response code: {response.status_code}, " + f"{response.reason}, {response.json()}" + ) + result = response.json() + if result.get("code"): + raise Exception(result["raw_log"]) + return result["balances"] def test_basic(network): pw3 = network.primary.w3 - rw3 = network.replica.w3 - validator = ADDRS["validator"] - community = ADDRS["community"] + pcli = network.primary.cosmos_cli() + validator = pcli.address("validator") + community = pcli.address("community") + print("address: ", validator, community) + replica_grpc_port = ports.api_port(network.replica.base_port(0)) + def print_balance(): - print(pw3.eth.get_balance(community)) - print(rw3.eth.get_balance(community)) + print("primary", pcli.balances(community), pcli.balances(validator)) + print("replica", grpc_call(replica_grpc_port, community), grpc_call(replica_grpc_port, validator)) print_balance() txhash = pw3.eth.send_transaction( { - "from": validator, - "to": community, + "from": ADDRS["validator"], + "to": ADDRS["community"], "value": 1000, } ) receipt = pw3.eth.wait_for_transaction_receipt(txhash) assert receipt.status == 1 + time.sleep(1) print_balance() From f16555ac287383c24f1db9419128a2cf7a8f41bc Mon Sep 17 00:00:00 2001 From: mmsqe Date: Sat, 22 Oct 2022 09:40:31 +0800 Subject: [PATCH 15/31] stop file server --- integration_tests/test_query_only.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/test_query_only.py b/integration_tests/test_query_only.py index b00399799f..66bb5d6433 100644 --- a/integration_tests/test_query_only.py +++ b/integration_tests/test_query_only.py @@ -77,6 +77,7 @@ def network(tmp_path_factory): wait_for_port(ports.grpc_port(base_port1)) yield Network(Cronos(path0 / chain_id), Cronos(path1 / chain_id)) finally: + httpd.shutdown() for proc in procs: print("killing:", proc.pid) os.killpg(os.getpgid(proc.pid), signal.SIGTERM) From f73463b122940201ed6a2df967413e99fc907a02 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Sat, 22 Oct 2022 09:41:17 +0800 Subject: [PATCH 16/31] add file check --- file/watcher.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/file/watcher.go b/file/watcher.go index 3eceb12b90..8bbf8c8e07 100644 --- a/file/watcher.go +++ b/file/watcher.go @@ -1,8 +1,10 @@ package file import ( + "encoding/binary" "errors" "fmt" + "io" "io/ioutil" "net/http" "os" @@ -104,6 +106,19 @@ func (w *BlockFileWatcher) SubscribeError() <-chan error { func (w *BlockFileWatcher) fetch(blockNum int) error { path := w.getFilePath(blockNum) + f, err := os.Open(path) + if err == nil { + defer f.Close() + var bytes [8]byte + if _, err = io.ReadFull(f, bytes[:]); err == nil { + size := binary.BigEndian.Uint64(bytes[:]) + if info, err := f.Stat(); err == nil && size+8 == uint64(info.Size()) { + return nil + } + } + f.Close() + } + // TBC: skip if exist path to avoid dup download data, err := w.downloader.GetData(path) fmt.Printf("mm-fetch: %+v, %+v, %+v\n", blockNum, len(data), err) From b732cfd4eaa046c220f24c494ffa374f17fa9ad0 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Mon, 24 Oct 2022 09:27:52 +0800 Subject: [PATCH 17/31] wait cross check balances --- integration_tests/configs/replica.jsonnet | 2 +- integration_tests/test_query_only.py | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/integration_tests/configs/replica.jsonnet b/integration_tests/configs/replica.jsonnet index 1d55df2020..803dd1ec52 100644 --- a/integration_tests/configs/replica.jsonnet +++ b/integration_tests/configs/replica.jsonnet @@ -9,7 +9,7 @@ config { 'is-local': false, 'remote-url': 'http://localhost:8080', 'grpc-only': true, - 'concurrency': 1, + 'concurrency': 6, }, }, } diff --git a/integration_tests/test_query_only.py b/integration_tests/test_query_only.py index 66bb5d6433..9449ddde8e 100644 --- a/integration_tests/test_query_only.py +++ b/integration_tests/test_query_only.py @@ -1,7 +1,6 @@ import os import signal import subprocess -import time from functools import partial from http.server import HTTPServer, SimpleHTTPRequestHandler from pathlib import Path @@ -13,7 +12,7 @@ from pystarport import ports from .network import Cronos -from .utils import ADDRS, wait_for_port +from .utils import ADDRS, wait_for_fn, wait_for_port class Network(NamedTuple): @@ -79,7 +78,6 @@ def network(tmp_path_factory): finally: httpd.shutdown() for proc in procs: - print("killing:", proc.pid) os.killpg(os.getpgid(proc.pid), signal.SIGTERM) proc.wait() print("killed:", proc.pid) @@ -107,11 +105,16 @@ def test_basic(network): print("address: ", validator, community) replica_grpc_port = ports.api_port(network.replica.base_port(0)) - def print_balance(): - print("primary", pcli.balances(community), pcli.balances(validator)) - print("replica", grpc_call(replica_grpc_port, community), grpc_call(replica_grpc_port, validator)) + def check_balances(): + pbalances = [pcli.balances(community), pcli.balances(validator)] + rbalances = [ + grpc_call(replica_grpc_port, community), + grpc_call(replica_grpc_port, validator), + ] + print("primary", pbalances) + print("replica", rbalances) + return pbalances == rbalances - print_balance() txhash = pw3.eth.send_transaction( { "from": ADDRS["validator"], @@ -121,5 +124,4 @@ def print_balance(): ) receipt = pw3.eth.wait_for_transaction_receipt(txhash) assert receipt.status == 1 - time.sleep(1) - print_balance() + wait_for_fn("cross-check-balances", check_balances, timeout=50) From bbc805344e47d65be9be2f247c7407b3e294cfa4 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Mon, 24 Oct 2022 09:28:34 +0800 Subject: [PATCH 18/31] chain ws to synchronizer and streamer subscribe latest blk via tm, sync historical via http --- app/app.go | 92 +++++++++++++++++++++------ client/ws.go | 95 ++++++++++++++++++++++++++++ file/watcher.go | 26 ++++++-- go.mod | 2 +- integration_tests/test_query_only.py | 11 ++-- 5 files changed, 196 insertions(+), 30 deletions(-) create mode 100644 client/ws.go diff --git a/app/app.go b/app/app.go index 6e80c128a6..b78491ceea 100644 --- a/app/app.go +++ b/app/app.go @@ -12,6 +12,7 @@ import ( "github.com/crypto-org-chain/cronos/x/cronos/middleware" "github.com/cosmos/cosmos-sdk/client" + "github.com/cosmos/cosmos-sdk/codec/types" "github.com/cosmos/cosmos-sdk/server" "github.com/gorilla/mux" @@ -132,6 +133,7 @@ import ( cronoskeeper "github.com/crypto-org-chain/cronos/x/cronos/keeper" evmhandlers "github.com/crypto-org-chain/cronos/x/cronos/keeper/evmhandlers" cronostypes "github.com/crypto-org-chain/cronos/x/cronos/types" + tmtypes "github.com/tendermint/tendermint/types" // unnamed import of statik for swagger UI support _ "github.com/crypto-org-chain/cronos/client/docs/statik" @@ -390,12 +392,38 @@ func New( startBlockNum = 0 } nextBlockNum := int(startBlockNum) + 1 + // TODO: maxBlockNum init from primary node + maxBlockNum := 0 + interval := time.Second + directory := filepath.Join(rootDir, "data", "file_streamer") + // streamer write the file blk by blk with concurrency 1 + streamer := cronosfile.NewBlockFileWatcher(1, maxBlockNum, func(blockNum int) string { + return cronosfile.GetLocalDataFileName(directory, blockNum) + }, true) + streamer.Start(nextBlockNum, interval) + go func() { + chData, chErr := streamer.SubscribeData(), streamer.SubscribeError() + for { + select { + case data := <-chData: + pairs, err := cronosfile.DecodeData(data.Data) + fmt.Printf("mm-pairs: %+v, %+v\n", len(pairs), err) + if err == nil { + versionDB.PutAtVersion(int64(data.BlockNum), pairs) + } + case err := <-chErr: + // fail read + fmt.Println("mm-fail-read-panic") + panic(err) + } + } + }() + isLocal := cast.ToBool(appOpts.Get(cronosappclient.FlagIsLocal)) remoteUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteUrl)) concurrency := cast.ToInt(appOpts.Get(cronosappclient.FlagConcurrency)) - interval := time.Second - synchronizer := cronosfile.NewBlockFileWatcher(concurrency, func(blockNum int) string { + synchronizer := cronosfile.NewBlockFileWatcher(concurrency, maxBlockNum, func(blockNum int) string { return fmt.Sprintf("%s/%s", remoteUrl, cronosfile.DataFileName(blockNum)) }, isLocal) synchronizer.Start(nextBlockNum, interval) @@ -408,43 +436,65 @@ func New( select { case data := <-chData: file := cronosfile.GetLocalDataFileName(directory, data.BlockNum) - fmt.Printf("mm-file: %+v\n", file) + fmt.Printf("mm-data.BlockNum: %+v\n", data.BlockNum) if err := os.WriteFile(file, data.Data, 0644); err != nil { + fmt.Println("mm-WriteFile-panic") panic(err) } retry = 0 - // fmt.Printf("mm-pairs: %+v\n", len(pairs)) - // versionDB.PutAtVersion(int64(data.BlockNum), pairs) + fmt.Println("mm-reset-retry") + if data.BlockNum > maxBlockNum { + streamer.SetMaxBlockNum(data.BlockNum) + } case err := <-chErr: retry++ + fmt.Println("mm-retry", retry) if retry == maxRetry { // data corrupt + fmt.Println("mm-data-corrupt-panic") panic(err) } } } }() - // streamer write the file blk by blk with concurrency 1 - streamer := cronosfile.NewBlockFileWatcher(1, func(blockNum int) string { - return cronosfile.GetLocalDataFileName(directory, blockNum) - }, true) - streamer.Start(nextBlockNum, interval) go func() { - chData, chErr := streamer.SubscribeData(), streamer.SubscribeError() - for { - select { - case data := <-chData: - pairs, err := cronosfile.DecodeData(data.Data) - fmt.Printf("mm-pairs: %+v, %+v\n", len(pairs), err) - if err == nil { - versionDB.PutAtVersion(int64(data.BlockNum), pairs) + maxRetry := 50 + for i := 0; i < maxRetry; i++ { + if i > 0 { + time.Sleep(time.Second) + } + // TODO: config remote tm + wsClient := cronosappclient.NewWebsocketClient("ws://localhost:26767/websocket") + chResult, err := wsClient.Subscribe() + if err != nil { + fmt.Printf("mm-subscribed[%+v]: %+v\n", i, err) + continue + } + fmt.Println("subscribing") + err = wsClient.Send("subscribe", []string{ + "tm.event='NewBlockHeader'", + }) + if err != nil { + fmt.Printf("mm-subscribed: %+v\n", err) + continue + } + i = 0 + fmt.Println("subscribed ws") + for res := range chResult { + if res == nil || res.Data == nil { + continue } - case err := <-chErr: - // fail read - panic(err) + data, ok := res.Data.(tmtypes.EventDataNewBlockHeader) + if !ok { + continue + } + blockNum := int(data.Header.Height) + fmt.Printf("mm-set-max-blk: %+v\n", blockNum) + synchronizer.SetMaxBlockNum(blockNum) } } + panic(fmt.Sprintf("max retries %d reached", maxRetry)) }() } diff --git a/client/ws.go b/client/ws.go new file mode 100644 index 0000000000..6aff2aa439 --- /dev/null +++ b/client/ws.go @@ -0,0 +1,95 @@ +package client + +import ( + "bytes" + "context" + "fmt" + "time" + + tmjson "github.com/tendermint/tendermint/libs/json" + coretypes "github.com/tendermint/tendermint/rpc/core/types" + types "github.com/tendermint/tendermint/rpc/jsonrpc/types" + "nhooyr.io/websocket" + "nhooyr.io/websocket/wsjson" +) + +type WebsocketClient struct { + url string + wsconn *websocket.Conn +} + +func NewWebsocketClient(url string) *WebsocketClient { + return &WebsocketClient{url: url} +} + +func (c *WebsocketClient) Subscribe() (<-chan *coretypes.ResultEvent, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + conn, _, err := websocket.Dial(ctx, c.url, nil) + if err != nil { + return nil, err + } + + c.wsconn = conn + conn.SetReadLimit(10240000) + + chResult := make(chan *coretypes.ResultEvent) + go func() { + defer close(chResult) + for { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + _, r, err := c.wsconn.Reader(ctx) + if err != nil { + cancel() + continue + } + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(r) + if err != nil { + cancel() + continue + } + cancel() + + bz := buf.Bytes() + res := new(types.RPCResponse) + err = tmjson.Unmarshal(bz, res) + if err != nil { + fmt.Printf("mm-read-res-err: %+v\n", err) + // break trigger close channel while continue will ignore the invalid response + break + } + ev := new(coretypes.ResultEvent) + if err := tmjson.Unmarshal(res.Result, ev); err != nil { + fmt.Printf("mm-read-ev-err: %+v\n", err) + break + } + chResult <- ev + time.Sleep(time.Second) + } + }() + return chResult, nil +} + +func (c *WebsocketClient) Send( + method string, + params []string, +) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + return wsjson.Write(ctx, c.wsconn, map[string]interface{}{ + "jsonrpc": "2.0", + "method": method, + "params": params, + "id": 1, + }) +} + +func (c *WebsocketClient) Close() { + if c.wsconn != nil { + c.wsconn.Close(websocket.StatusNormalClosure, "") + c.wsconn = nil + } +} diff --git a/file/watcher.go b/file/watcher.go index 8bbf8c8e07..3d53fbf68b 100644 --- a/file/watcher.go +++ b/file/watcher.go @@ -9,6 +9,7 @@ import ( "net/http" "os" "sync" + "sync/atomic" "time" ) @@ -60,6 +61,7 @@ type BlockData struct { type BlockFileWatcher struct { concurrency int + maxBlockNum int64 getFilePath func(blockNum int) string downloader fileDownloader chData chan *BlockData @@ -70,11 +72,13 @@ type BlockFileWatcher struct { func NewBlockFileWatcher( concurrency int, + maxBlockNum int, getFilePath func(blockNum int) string, isLocal bool, ) *BlockFileWatcher { w := &BlockFileWatcher{ concurrency: concurrency, + maxBlockNum: int64(maxBlockNum), getFilePath: getFilePath, chData: make(chan *BlockData), chError: make(chan error), @@ -104,11 +108,16 @@ func (w *BlockFileWatcher) SubscribeError() <-chan error { return w.chError } +func (w *BlockFileWatcher) SetMaxBlockNum(num int) { + atomic.StoreInt64(&w.maxBlockNum, int64(num)) +} + func (w *BlockFileWatcher) fetch(blockNum int) error { path := w.getFilePath(blockNum) f, err := os.Open(path) if err == nil { defer f.Close() + // valid 1st 8 bytes for downloaded file var bytes [8]byte if _, err = io.ReadFull(f, bytes[:]); err == nil { size := binary.BigEndian.Uint64(bytes[:]) @@ -119,7 +128,7 @@ func (w *BlockFileWatcher) fetch(blockNum int) error { f.Close() } - // TBC: skip if exist path to avoid dup download + // download if file not exist data, err := w.downloader.GetData(path) fmt.Printf("mm-fetch: %+v, %+v, %+v\n", blockNum, len(data), err) if err != nil { @@ -158,8 +167,18 @@ func (w *BlockFileWatcher) Start( default: wg := new(sync.WaitGroup) - resultErrs := make([]error, w.concurrency) - for i := 0; i < w.concurrency; i++ { + currentBlockNum := blockNum + maxBlockNum := int(atomic.LoadInt64(&w.maxBlockNum)) + concurrency := w.concurrency + if diff := maxBlockNum - currentBlockNum; diff < concurrency { + if diff <= 0 { + time.Sleep(interval) + break + } + concurrency = diff + } + resultErrs := make([]error, concurrency) + for i := 0; i < concurrency; i++ { nextBlockNum := blockNum + i fmt.Println("mm-start: ", nextBlockNum) if !finishedBlockNums[nextBlockNum] { @@ -173,7 +192,6 @@ func (w *BlockFileWatcher) Start( } wg.Wait() errReached := false - currentBlockNum := blockNum for i, err := range resultErrs { b := currentBlockNum + i if err != nil { diff --git a/go.mod b/go.mod index 78ff2d7709..5ce5150e08 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( google.golang.org/grpc v1.50.1 google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8 gopkg.in/yaml.v2 v2.4.0 + nhooyr.io/websocket v1.8.6 ) require ( @@ -188,7 +189,6 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - nhooyr.io/websocket v1.8.6 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/integration_tests/test_query_only.py b/integration_tests/test_query_only.py index 9449ddde8e..beb7ee2235 100644 --- a/integration_tests/test_query_only.py +++ b/integration_tests/test_query_only.py @@ -73,6 +73,7 @@ def network(tmp_path_factory): procs.append(exec(base / "replica.jsonnet", path1, base_port1)) try: wait_for_port(ports.evmrpc_port(base_port0)) + wait_for_port(ports.evmrpc_ws_port(base_port0)) wait_for_port(ports.grpc_port(base_port1)) yield Network(Cronos(path0 / chain_id), Cronos(path1 / chain_id)) finally: @@ -87,10 +88,12 @@ def grpc_call(p, address): url = f"http://127.0.0.1:{p}/cosmos/bank/v1beta1/balances/{address}" response = requests.get(url) if not response.ok: - raise Exception( - f"response code: {response.status_code}, " - f"{response.reason}, {response.json()}" - ) + # retry until file get synced + return -1 + # raise Exception( + # f"response code: {response.status_code}, " + # f"{response.reason}, {response.json()}" + # ) result = response.json() if result.get("code"): raise Exception(result["raw_log"]) From b24dfcb09b9f8016edcc281e5be96f7e73984914 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Wed, 26 Oct 2022 09:52:38 +0800 Subject: [PATCH 19/31] fix build --- app/app.go | 2 +- {file => client/file}/decoder.go | 0 {file => client/file}/watcher.go | 0 {file => client/file}/watcher_test.go | 0 versiondb/tmdb/store_test.go | 2 +- 5 files changed, 2 insertions(+), 2 deletions(-) rename {file => client/file}/decoder.go (100%) rename {file => client/file}/watcher.go (100%) rename {file => client/file}/watcher_test.go (100%) diff --git a/app/app.go b/app/app.go index b78491ceea..f4bad18d8e 100644 --- a/app/app.go +++ b/app/app.go @@ -126,7 +126,7 @@ import ( // this line is used by starport scaffolding # stargate/app/moduleImport cronosappclient "github.com/crypto-org-chain/cronos/client" - cronosfile "github.com/crypto-org-chain/cronos/file" + cronosfile "github.com/crypto-org-chain/cronos/client/file" "github.com/crypto-org-chain/cronos/versiondb" "github.com/crypto-org-chain/cronos/versiondb/tmdb" cronosclient "github.com/crypto-org-chain/cronos/x/cronos/client" diff --git a/file/decoder.go b/client/file/decoder.go similarity index 100% rename from file/decoder.go rename to client/file/decoder.go diff --git a/file/watcher.go b/client/file/watcher.go similarity index 100% rename from file/watcher.go rename to client/file/watcher.go diff --git a/file/watcher_test.go b/client/file/watcher_test.go similarity index 100% rename from file/watcher_test.go rename to client/file/watcher_test.go diff --git a/versiondb/tmdb/store_test.go b/versiondb/tmdb/store_test.go index ea7aee0b38..702cfe739e 100644 --- a/versiondb/tmdb/store_test.go +++ b/versiondb/tmdb/store_test.go @@ -6,7 +6,7 @@ import ( "github.com/cosmos/cosmos-sdk/codec" codectypes "github.com/cosmos/cosmos-sdk/codec/types" - "github.com/crypto-org-chain/cronos/file" + "github.com/crypto-org-chain/cronos/client/file" "github.com/crypto-org-chain/cronos/versiondb" "github.com/crypto-org-chain/cronos/x/cronos/types" "github.com/stretchr/testify/require" From 0eb6bf76a3021a328b7b35579ede57e7efc33d8e Mon Sep 17 00:00:00 2001 From: mmsqe Date: Wed, 26 Oct 2022 09:52:44 +0800 Subject: [PATCH 20/31] configurable flags --- app/app.go | 10 +++++----- client/flags.go | 2 ++ integration_tests/configs/replica.jsonnet | 4 +++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/app/app.go b/app/app.go index f4bad18d8e..edeb0989f5 100644 --- a/app/app.go +++ b/app/app.go @@ -382,6 +382,7 @@ func New( versionDB := tmdb.NewStore(plainDB, historyDB, changesetDB) isGrpcOnly := cast.ToBool(appOpts.Get(cronosappclient.FlagIsGrpcOnly)) if isGrpcOnly { + const defaultMaxRetry = 50 latestVersion, err := versionDB.GetLatestVersion() fmt.Printf("mm-latestVersion: %+v\n", latestVersion) if err != nil { @@ -422,6 +423,7 @@ func New( isLocal := cast.ToBool(appOpts.Get(cronosappclient.FlagIsLocal)) remoteUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteUrl)) + remoteWsUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteWsUrl)) concurrency := cast.ToInt(appOpts.Get(cronosappclient.FlagConcurrency)) synchronizer := cronosfile.NewBlockFileWatcher(concurrency, maxBlockNum, func(blockNum int) string { return fmt.Sprintf("%s/%s", remoteUrl, cronosfile.DataFileName(blockNum)) @@ -459,13 +461,11 @@ func New( }() go func() { - maxRetry := 50 - for i := 0; i < maxRetry; i++ { + for i := 0; i < defaultMaxRetry; i++ { if i > 0 { time.Sleep(time.Second) } - // TODO: config remote tm - wsClient := cronosappclient.NewWebsocketClient("ws://localhost:26767/websocket") + wsClient := cronosappclient.NewWebsocketClient(remoteWsUrl) chResult, err := wsClient.Subscribe() if err != nil { fmt.Printf("mm-subscribed[%+v]: %+v\n", i, err) @@ -494,7 +494,7 @@ func New( synchronizer.SetMaxBlockNum(blockNum) } } - panic(fmt.Sprintf("max retries %d reached", maxRetry)) + panic(fmt.Sprintf("max retries %d reached", defaultMaxRetry)) }() } diff --git a/client/flags.go b/client/flags.go index 48d00913de..4803540414 100644 --- a/client/flags.go +++ b/client/flags.go @@ -5,4 +5,6 @@ const FlagIsGrpcOnly = "grpc-only" const FlagIsLocal = "is-local" const FlagLocalDirectory = "local-directory" const FlagRemoteUrl = "remote-url" +const FlagRemoteGrpcUrl = "remote-grpc-url" +const FlagRemoteWsUrl = "remote-ws-url" const FlagConcurrency = "concurrency" diff --git a/integration_tests/configs/replica.jsonnet b/integration_tests/configs/replica.jsonnet index 803dd1ec52..920425a750 100644 --- a/integration_tests/configs/replica.jsonnet +++ b/integration_tests/configs/replica.jsonnet @@ -7,7 +7,9 @@ config { enable: false, }, 'is-local': false, - 'remote-url': 'http://localhost:8080', + 'remote-url': 'http://0.0.0.0:8080', + 'remote-grpc-url': 'http://0.0.0.0:26754', + 'remote-ws-url': 'ws://0.0.0.0:26767/websocket', 'grpc-only': true, 'concurrency': 6, }, From 58476f2225b8840cae52ddbe43472b8374fa2c0c Mon Sep 17 00:00:00 2001 From: mmsqe Date: Wed, 26 Oct 2022 09:52:48 +0800 Subject: [PATCH 21/31] get max blk num from primary --- app/app.go | 51 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/app/app.go b/app/app.go index edeb0989f5..e0682e8a4e 100644 --- a/app/app.go +++ b/app/app.go @@ -3,6 +3,7 @@ package app import ( "fmt" "io" + "io/ioutil" "net/http" "os" "path/filepath" @@ -310,6 +311,19 @@ type App struct { experimental bool } +// Simplify block height for header +type Header struct { + Height int64 `json:"height,omitempty"` +} + +type Block struct { + Header Header `json:"header"` +} + +type GetLatestBlockResponse struct { + Block *Block `json:"block,omitempty"` +} + // New returns a reference to an initialized chain. // NewSimApp returns a reference to an initialized SimApp. func New( @@ -393,10 +407,41 @@ func New( startBlockNum = 0 } nextBlockNum := int(startBlockNum) + 1 - // TODO: maxBlockNum init from primary node - maxBlockNum := 0 - interval := time.Second + remoteGrpcUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteGrpcUrl)) + maxBlockNum := -1 + for i := 0; i < defaultMaxRetry; i++ { + if i > 0 { + time.Sleep(time.Second) + } + resp, err := http.Get(fmt.Sprintf("%s/cosmos/base/tendermint/v1beta1/blocks/latest", remoteGrpcUrl)) + if err != nil { + fmt.Printf("error making http request: %s\n", err) + continue + } + bz, err := ioutil.ReadAll(resp.Body) + if err != nil { + continue + } + if resp.StatusCode != http.StatusOK { + continue + } + resp.Body.Close() + res := new(GetLatestBlockResponse) + err = tmjson.Unmarshal(bz, res) + if err != nil { + fmt.Printf("mm-read-res-err: %+v\n", err) + continue + } + maxBlockNum = int(res.Block.Header.Height) + fmt.Printf("mm-maxBlockNum: %d\n", maxBlockNum) + break + } + if maxBlockNum < 0 { + panic(fmt.Sprintf("max retries %d reached", defaultMaxRetry)) + } + + interval := time.Second directory := filepath.Join(rootDir, "data", "file_streamer") // streamer write the file blk by blk with concurrency 1 streamer := cronosfile.NewBlockFileWatcher(1, maxBlockNum, func(blockNum int) string { From 1f4fc87367fd100d66c0f4b3f5b723d594a1ba3f Mon Sep 17 00:00:00 2001 From: mmsqe Date: Wed, 26 Oct 2022 09:52:53 +0800 Subject: [PATCH 22/31] separate app from sync --- app/app.go | 169 ++----------------------------------------- client/file/sync.go | 173 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+), 163 deletions(-) create mode 100644 client/file/sync.go diff --git a/app/app.go b/app/app.go index e0682e8a4e..e6daccc53f 100644 --- a/app/app.go +++ b/app/app.go @@ -3,17 +3,14 @@ package app import ( "fmt" "io" - "io/ioutil" "net/http" "os" "path/filepath" - "time" "github.com/crypto-org-chain/cronos/x/cronos" "github.com/crypto-org-chain/cronos/x/cronos/middleware" "github.com/cosmos/cosmos-sdk/client" - "github.com/cosmos/cosmos-sdk/codec/types" "github.com/cosmos/cosmos-sdk/server" "github.com/gorilla/mux" @@ -134,7 +131,6 @@ import ( cronoskeeper "github.com/crypto-org-chain/cronos/x/cronos/keeper" evmhandlers "github.com/crypto-org-chain/cronos/x/cronos/keeper/evmhandlers" cronostypes "github.com/crypto-org-chain/cronos/x/cronos/types" - tmtypes "github.com/tendermint/tendermint/types" // unnamed import of statik for swagger UI support _ "github.com/crypto-org-chain/cronos/client/docs/statik" @@ -311,19 +307,6 @@ type App struct { experimental bool } -// Simplify block height for header -type Header struct { - Height int64 `json:"height,omitempty"` -} - -type Block struct { - Header Header `json:"header"` -} - -type GetLatestBlockResponse struct { - Block *Block `json:"block,omitempty"` -} - // New returns a reference to an initialized chain. // NewSimApp returns a reference to an initialized SimApp. func New( @@ -395,154 +378,14 @@ func New( } versionDB := tmdb.NewStore(plainDB, historyDB, changesetDB) isGrpcOnly := cast.ToBool(appOpts.Get(cronosappclient.FlagIsGrpcOnly)) + remoteGrpcUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteGrpcUrl)) + isLocal := cast.ToBool(appOpts.Get(cronosappclient.FlagIsLocal)) + remoteUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteUrl)) + remoteWsUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteWsUrl)) + concurrency := cast.ToInt(appOpts.Get(cronosappclient.FlagConcurrency)) if isGrpcOnly { - const defaultMaxRetry = 50 - latestVersion, err := versionDB.GetLatestVersion() - fmt.Printf("mm-latestVersion: %+v\n", latestVersion) - if err != nil { - panic(err) - } - startBlockNum := latestVersion - if startBlockNum < 0 { - startBlockNum = 0 - } - nextBlockNum := int(startBlockNum) + 1 - remoteGrpcUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteGrpcUrl)) - maxBlockNum := -1 - for i := 0; i < defaultMaxRetry; i++ { - if i > 0 { - time.Sleep(time.Second) - } - resp, err := http.Get(fmt.Sprintf("%s/cosmos/base/tendermint/v1beta1/blocks/latest", remoteGrpcUrl)) - if err != nil { - fmt.Printf("error making http request: %s\n", err) - continue - } - - bz, err := ioutil.ReadAll(resp.Body) - if err != nil { - continue - } - if resp.StatusCode != http.StatusOK { - continue - } - resp.Body.Close() - res := new(GetLatestBlockResponse) - err = tmjson.Unmarshal(bz, res) - if err != nil { - fmt.Printf("mm-read-res-err: %+v\n", err) - continue - } - maxBlockNum = int(res.Block.Header.Height) - fmt.Printf("mm-maxBlockNum: %d\n", maxBlockNum) - break - } - if maxBlockNum < 0 { - panic(fmt.Sprintf("max retries %d reached", defaultMaxRetry)) - } - - interval := time.Second - directory := filepath.Join(rootDir, "data", "file_streamer") - // streamer write the file blk by blk with concurrency 1 - streamer := cronosfile.NewBlockFileWatcher(1, maxBlockNum, func(blockNum int) string { - return cronosfile.GetLocalDataFileName(directory, blockNum) - }, true) - streamer.Start(nextBlockNum, interval) - go func() { - chData, chErr := streamer.SubscribeData(), streamer.SubscribeError() - for { - select { - case data := <-chData: - pairs, err := cronosfile.DecodeData(data.Data) - fmt.Printf("mm-pairs: %+v, %+v\n", len(pairs), err) - if err == nil { - versionDB.PutAtVersion(int64(data.BlockNum), pairs) - } - case err := <-chErr: - // fail read - fmt.Println("mm-fail-read-panic") - panic(err) - } - } - }() - - isLocal := cast.ToBool(appOpts.Get(cronosappclient.FlagIsLocal)) - remoteUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteUrl)) - remoteWsUrl := cast.ToString(appOpts.Get(cronosappclient.FlagRemoteWsUrl)) - concurrency := cast.ToInt(appOpts.Get(cronosappclient.FlagConcurrency)) - synchronizer := cronosfile.NewBlockFileWatcher(concurrency, maxBlockNum, func(blockNum int) string { - return fmt.Sprintf("%s/%s", remoteUrl, cronosfile.DataFileName(blockNum)) - }, isLocal) - synchronizer.Start(nextBlockNum, interval) - go func() { - // max retry for temporary io error - maxRetry := concurrency * 2 - retry := 0 - chData, chErr := synchronizer.SubscribeData(), synchronizer.SubscribeError() - for { - select { - case data := <-chData: - file := cronosfile.GetLocalDataFileName(directory, data.BlockNum) - fmt.Printf("mm-data.BlockNum: %+v\n", data.BlockNum) - if err := os.WriteFile(file, data.Data, 0644); err != nil { - fmt.Println("mm-WriteFile-panic") - panic(err) - } - retry = 0 - fmt.Println("mm-reset-retry") - if data.BlockNum > maxBlockNum { - streamer.SetMaxBlockNum(data.BlockNum) - } - case err := <-chErr: - retry++ - fmt.Println("mm-retry", retry) - if retry == maxRetry { - // data corrupt - fmt.Println("mm-data-corrupt-panic") - panic(err) - } - } - } - }() - - go func() { - for i := 0; i < defaultMaxRetry; i++ { - if i > 0 { - time.Sleep(time.Second) - } - wsClient := cronosappclient.NewWebsocketClient(remoteWsUrl) - chResult, err := wsClient.Subscribe() - if err != nil { - fmt.Printf("mm-subscribed[%+v]: %+v\n", i, err) - continue - } - fmt.Println("subscribing") - err = wsClient.Send("subscribe", []string{ - "tm.event='NewBlockHeader'", - }) - if err != nil { - fmt.Printf("mm-subscribed: %+v\n", err) - continue - } - i = 0 - fmt.Println("subscribed ws") - for res := range chResult { - if res == nil || res.Data == nil { - continue - } - data, ok := res.Data.(tmtypes.EventDataNewBlockHeader) - if !ok { - continue - } - blockNum := int(data.Header.Height) - fmt.Printf("mm-set-max-blk: %+v\n", blockNum) - synchronizer.SetMaxBlockNum(blockNum) - } - } - panic(fmt.Sprintf("max retries %d reached", defaultMaxRetry)) - }() + cronosfile.Sync(versionDB, remoteGrpcUrl, remoteUrl, remoteWsUrl, rootDir, isLocal, concurrency) } - // default to exposing all exposeStoreKeys := make([]storetypes.StoreKey, 0, len(keys)) for _, storeKey := range keys { diff --git a/client/file/sync.go b/client/file/sync.go new file mode 100644 index 0000000000..e410785059 --- /dev/null +++ b/client/file/sync.go @@ -0,0 +1,173 @@ +package file + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/crypto-org-chain/cronos/client" + "github.com/crypto-org-chain/cronos/versiondb/tmdb" + tmjson "github.com/tendermint/tendermint/libs/json" + tmtypes "github.com/tendermint/tendermint/types" +) + +// Simplify block height for header +type Header struct { + Height int64 `json:"height,omitempty"` +} + +type Block struct { + Header Header `json:"header"` +} + +type GetLatestBlockResponse struct { + Block *Block `json:"block,omitempty"` +} + +func Sync(versionDB *tmdb.Store, remoteGrpcUrl, remoteUrl, remoteWsUrl, rootDir string, isLocal bool, concurrency int) { + + const defaultMaxRetry = 50 + latestVersion, err := versionDB.GetLatestVersion() + fmt.Printf("mm-latestVersion: %+v\n", latestVersion) + if err != nil { + panic(err) + } + startBlockNum := latestVersion + if startBlockNum < 0 { + startBlockNum = 0 + } + nextBlockNum := int(startBlockNum) + 1 + maxBlockNum := -1 + for i := 0; i < defaultMaxRetry; i++ { + if i > 0 { + time.Sleep(time.Second) + } + resp, err := http.Get(fmt.Sprintf("%s/cosmos/base/tendermint/v1beta1/blocks/latest", remoteGrpcUrl)) + if err != nil { + fmt.Printf("error making http request: %s\n", err) + continue + } + + bz, err := ioutil.ReadAll(resp.Body) + if err != nil { + continue + } + if resp.StatusCode != http.StatusOK { + continue + } + resp.Body.Close() + res := new(GetLatestBlockResponse) + err = tmjson.Unmarshal(bz, res) + if err != nil { + fmt.Printf("mm-read-res-err: %+v\n", err) + continue + } + maxBlockNum = int(res.Block.Header.Height) + fmt.Printf("mm-maxBlockNum: %d\n", maxBlockNum) + break + } + if maxBlockNum < 0 { + panic(fmt.Sprintf("max retries %d reached", defaultMaxRetry)) + } + + interval := time.Second + directory := filepath.Join(rootDir, "data", "file_streamer") + // streamer write the file blk by blk with concurrency 1 + streamer := NewBlockFileWatcher(1, maxBlockNum, func(blockNum int) string { + return GetLocalDataFileName(directory, blockNum) + }, true) + streamer.Start(nextBlockNum, interval) + go func() { + chData, chErr := streamer.SubscribeData(), streamer.SubscribeError() + for { + select { + case data := <-chData: + pairs, err := DecodeData(data.Data) + fmt.Printf("mm-pairs: %+v, %+v\n", len(pairs), err) + if err == nil { + versionDB.PutAtVersion(int64(data.BlockNum), pairs) + } + case err := <-chErr: + // fail read + fmt.Println("mm-fail-read-panic") + panic(err) + } + } + }() + + synchronizer := NewBlockFileWatcher(concurrency, maxBlockNum, func(blockNum int) string { + return fmt.Sprintf("%s/%s", remoteUrl, DataFileName(blockNum)) + }, isLocal) + synchronizer.Start(nextBlockNum, interval) + go func() { + // max retry for temporary io error + maxRetry := concurrency * 2 + retry := 0 + chData, chErr := synchronizer.SubscribeData(), synchronizer.SubscribeError() + for { + select { + case data := <-chData: + file := GetLocalDataFileName(directory, data.BlockNum) + fmt.Printf("mm-data.BlockNum: %+v\n", data.BlockNum) + if err := os.WriteFile(file, data.Data, 0644); err != nil { + fmt.Println("mm-WriteFile-panic") + panic(err) + } + retry = 0 + fmt.Println("mm-reset-retry") + if data.BlockNum > maxBlockNum { + streamer.SetMaxBlockNum(data.BlockNum) + } + case err := <-chErr: + retry++ + fmt.Println("mm-retry", retry) + if retry == maxRetry { + // data corrupt + fmt.Println("mm-data-corrupt-panic") + panic(err) + } + } + } + }() + + go func() { + for i := 0; i < defaultMaxRetry; i++ { + if i > 0 { + time.Sleep(time.Second) + } + wsClient := client.NewWebsocketClient(remoteWsUrl) + chResult, err := wsClient.Subscribe() + if err != nil { + fmt.Printf("mm-subscribed[%+v]: %+v\n", i, err) + continue + } + fmt.Println("subscribing") + err = wsClient.Send("subscribe", []string{ + "tm.event='NewBlockHeader'", + }) + if err != nil { + fmt.Printf("mm-subscribed: %+v\n", err) + continue + } + i = 0 + fmt.Println("subscribed ws") + for res := range chResult { + if res == nil || res.Data == nil { + continue + } + data, ok := res.Data.(tmtypes.EventDataNewBlockHeader) + if !ok { + continue + } + blockNum := int(data.Header.Height) + fmt.Printf("mm-set-max-blk: %+v\n", blockNum) + synchronizer.SetMaxBlockNum(blockNum) + streamer.SetMaxBlockNum(blockNum) + } + } + panic(fmt.Sprintf("max retries %d reached", defaultMaxRetry)) + }() +} From b8b75f79ab0afcf9378a412053e7d098d6dbe5dc Mon Sep 17 00:00:00 2001 From: mmsqe Date: Wed, 26 Oct 2022 09:52:57 +0800 Subject: [PATCH 23/31] handle close inside loop --- client/file/sync.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/client/file/sync.go b/client/file/sync.go index e410785059..1b51ad45de 100644 --- a/client/file/sync.go +++ b/client/file/sync.go @@ -51,14 +51,21 @@ func Sync(versionDB *tmdb.Store, remoteGrpcUrl, remoteUrl, remoteWsUrl, rootDir continue } - bz, err := ioutil.ReadAll(resp.Body) - if err != nil { - continue - } - if resp.StatusCode != http.StatusOK { + var bz []byte + if result := func() bool { + defer resp.Body.Close() + bz, err = ioutil.ReadAll(resp.Body) + if err != nil { + return false + } + if resp.StatusCode != http.StatusOK { + return false + } + return true + }(); !result { continue } - resp.Body.Close() + res := new(GetLatestBlockResponse) err = tmjson.Unmarshal(bz, res) if err != nil { From 7518c5ef8887ffacacad471df0a7195b09a77be6 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Wed, 26 Oct 2022 09:53:03 +0800 Subject: [PATCH 24/31] only check the file size, the file is not corrupted even close error --- client/file/watcher.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/file/watcher.go b/client/file/watcher.go index 3d53fbf68b..739017ea2e 100644 --- a/client/file/watcher.go +++ b/client/file/watcher.go @@ -116,7 +116,7 @@ func (w *BlockFileWatcher) fetch(blockNum int) error { path := w.getFilePath(blockNum) f, err := os.Open(path) if err == nil { - defer f.Close() + defer f.Close() //nolint // valid 1st 8 bytes for downloaded file var bytes [8]byte if _, err = io.ReadFull(f, bytes[:]); err == nil { @@ -125,7 +125,7 @@ func (w *BlockFileWatcher) fetch(blockNum int) error { return nil } } - f.Close() + f.Close() //nolint } // download if file not exist From 591d60421d49a5f3373bd9f0cc5664343c499540 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Wed, 26 Oct 2022 12:18:59 +0800 Subject: [PATCH 25/31] fix test --- client/{file => }/decoder.go | 2 +- client/file/sync.go | 2 +- client/file/watcher_test.go | 4 +- versiondb/backend_test_utils.go | 82 +++++++++++++++++---------------- versiondb/tmdb/store_test.go | 4 +- 5 files changed, 48 insertions(+), 46 deletions(-) rename client/{file => }/decoder.go (96%) diff --git a/client/file/decoder.go b/client/decoder.go similarity index 96% rename from client/file/decoder.go rename to client/decoder.go index 150d1f0be4..330ee32d65 100644 --- a/client/file/decoder.go +++ b/client/decoder.go @@ -1,4 +1,4 @@ -package file +package client import ( "github.com/cosmos/cosmos-sdk/store/types" diff --git a/client/file/sync.go b/client/file/sync.go index 1b51ad45de..9de708f489 100644 --- a/client/file/sync.go +++ b/client/file/sync.go @@ -92,7 +92,7 @@ func Sync(versionDB *tmdb.Store, remoteGrpcUrl, remoteUrl, remoteWsUrl, rootDir for { select { case data := <-chData: - pairs, err := DecodeData(data.Data) + pairs, err := client.DecodeData(data.Data) fmt.Printf("mm-pairs: %+v, %+v\n", len(pairs), err) if err == nil { versionDB.PutAtVersion(int64(data.BlockNum), pairs) diff --git a/client/file/watcher_test.go b/client/file/watcher_test.go index eee2204f40..51d2ad3096 100644 --- a/client/file/watcher_test.go +++ b/client/file/watcher_test.go @@ -53,7 +53,7 @@ func TestFileWatcher(t *testing.T) { t.Run("when sync via local", func(t *testing.T) { setupBlockFiles(directory, startBlockNum, endBlockNum) - watcher := NewBlockFileWatcher(concurrency, func(blockNum int) string { + watcher := NewBlockFileWatcher(concurrency, endBlockNum+1, func(blockNum int) string { return GetLocalDataFileName(directory, blockNum) }, true) total := start(watcher, startBlockNum, endBlockNum) @@ -69,7 +69,7 @@ func TestFileWatcher(t *testing.T) { go func() { log.Fatal(http.ListenAndServe(":"+port, nil)) }() - watcher := NewBlockFileWatcher(concurrency, func(blockNum int) string { + watcher := NewBlockFileWatcher(concurrency, endBlockNum+1, func(blockNum int) string { return fmt.Sprintf("http://localhost:%s/%s", port, DataFileName(blockNum)) }, false) total := start(watcher, startBlockNum, endBlockNum) diff --git a/versiondb/backend_test_utils.go b/versiondb/backend_test_utils.go index a20f6caacc..75522fca76 100644 --- a/versiondb/backend_test_utils.go +++ b/versiondb/backend_test_utils.go @@ -12,7 +12,9 @@ import ( var ( key1 = []byte("key1") - value1 = []byte("value1") + key2 = []byte("key2") + value1 = []byte("1") + value2 = []byte("2") key1Subkey = []byte("key1/subkey") ) @@ -20,30 +22,30 @@ func MockStoreKVPairs(v int64) []types.StoreKVPair { switch v { case 0: return []types.StoreKVPair{ - {StoreKey: "evm", Key: []byte("delete-in-block2"), Value: []byte("1")}, - {StoreKey: "evm", Key: []byte("re-add-in-block3"), Value: []byte("1")}, - {StoreKey: "evm", Key: []byte("z-genesis-only"), Value: []byte("2")}, - {StoreKey: "evm", Key: []byte("modify-in-block2"), Value: []byte("1")}, - {StoreKey: "staking", Key: []byte("key1"), Value: []byte("value1")}, - {StoreKey: "staking", Key: []byte("key1/subkey"), Value: []byte("value1")}, + {StoreKey: "evm", Key: []byte("delete-in-block2"), Value: value1}, + {StoreKey: "evm", Key: []byte("re-add-in-block3"), Value: value1}, + {StoreKey: "evm", Key: []byte("z-genesis-only"), Value: value2}, + {StoreKey: "evm", Key: []byte("modify-in-block2"), Value: value1}, + {StoreKey: "staking", Key: key1, Value: value1}, + {StoreKey: "staking", Key: key1Subkey, Value: value1}, } case 1: return []types.StoreKVPair{ {StoreKey: "evm", Key: []byte("re-add-in-block3"), Delete: true}, - {StoreKey: "evm", Key: []byte("add-in-block1"), Value: []byte("1")}, - {StoreKey: "staking", Key: []byte("key1"), Delete: true}, + {StoreKey: "evm", Key: []byte("add-in-block1"), Value: value1}, + {StoreKey: "staking", Key: key1, Delete: true}, } case 2: return []types.StoreKVPair{ - {StoreKey: "evm", Key: []byte("add-in-block2"), Value: []byte("1")}, + {StoreKey: "evm", Key: []byte("add-in-block2"), Value: value1}, {StoreKey: "evm", Key: []byte("delete-in-block2"), Delete: true}, - {StoreKey: "evm", Key: []byte("modify-in-block2"), Value: []byte("2")}, - {StoreKey: "evm", Key: []byte("key2"), Delete: true}, - {StoreKey: "staking", Key: []byte("key1"), Value: []byte("value2")}, + {StoreKey: "evm", Key: []byte("modify-in-block2"), Value: value2}, + {StoreKey: "evm", Key: key2, Delete: true}, + {StoreKey: "staking", Key: key1, Value: value2}, } case 3: return []types.StoreKVPair{ - {StoreKey: "evm", Key: []byte("re-add-in-block3"), Value: []byte("2")}, + {StoreKey: "evm", Key: []byte("re-add-in-block3"), Value: value2}, } case 4: return []types.StoreKVPair{ @@ -79,7 +81,7 @@ func testBasics(t *testing.T, store VersionStore) { value, err := store.GetAtVersion("evm", []byte("z-genesis-only"), nil) require.NoError(t, err) - require.Equal(t, value, []byte("2")) + require.Equal(t, value, value2) v = 4 ok, err := store.HasAtVersion("evm", []byte("z-genesis-only"), &v) @@ -87,7 +89,7 @@ func testBasics(t *testing.T, store VersionStore) { require.True(t, ok) value, err = store.GetAtVersion("evm", []byte("z-genesis-only"), &v) require.NoError(t, err) - require.Equal(t, value, []byte("2")) + require.Equal(t, value, value2) value, err = store.GetAtVersion("evm", []byte("re-add-in-block3"), nil) require.NoError(t, err) @@ -99,12 +101,12 @@ func testBasics(t *testing.T, store VersionStore) { value, err = store.GetAtVersion("staking", key1, nil) require.NoError(t, err) - require.Equal(t, value, []byte("value2")) + require.Equal(t, value, value2) v = 2 value, err = store.GetAtVersion("staking", key1, &v) require.NoError(t, err) - require.Equal(t, value, []byte("value2")) + require.Equal(t, value, value2) ok, err = store.HasAtVersion("staking", key1, &v) require.NoError(t, err) @@ -113,7 +115,7 @@ func testBasics(t *testing.T, store VersionStore) { v = 0 value, err = store.GetAtVersion("staking", key1, &v) require.NoError(t, err) - require.Equal(t, value, []byte("value1")) + require.Equal(t, value, value1) v = 1 value, err = store.GetAtVersion("staking", key1, &v) @@ -143,35 +145,35 @@ func testIterator(t *testing.T, store VersionStore) { expItems := [][]KVPair{ { - KVPair{[]byte("delete-in-block2"), []byte("1")}, - KVPair{[]byte("modify-in-block2"), []byte("1")}, - KVPair{[]byte("re-add-in-block3"), []byte("1")}, - KVPair{[]byte("z-genesis-only"), []byte("2")}, + KVPair{[]byte("delete-in-block2"), value1}, + KVPair{[]byte("modify-in-block2"), value1}, + KVPair{[]byte("re-add-in-block3"), value1}, + KVPair{[]byte("z-genesis-only"), value2}, }, { - KVPair{[]byte("add-in-block1"), []byte("1")}, - KVPair{[]byte("delete-in-block2"), []byte("1")}, - KVPair{[]byte("modify-in-block2"), []byte("1")}, - KVPair{[]byte("z-genesis-only"), []byte("2")}, + KVPair{[]byte("add-in-block1"), value1}, + KVPair{[]byte("delete-in-block2"), value1}, + KVPair{[]byte("modify-in-block2"), value1}, + KVPair{[]byte("z-genesis-only"), value2}, }, { - KVPair{[]byte("add-in-block1"), []byte("1")}, - KVPair{[]byte("add-in-block2"), []byte("1")}, - KVPair{[]byte("modify-in-block2"), []byte("2")}, - KVPair{[]byte("z-genesis-only"), []byte("2")}, + KVPair{[]byte("add-in-block1"), value1}, + KVPair{[]byte("add-in-block2"), value1}, + KVPair{[]byte("modify-in-block2"), value2}, + KVPair{[]byte("z-genesis-only"), value2}, }, { - KVPair{[]byte("add-in-block1"), []byte("1")}, - KVPair{[]byte("add-in-block2"), []byte("1")}, - KVPair{[]byte("modify-in-block2"), []byte("2")}, - KVPair{[]byte("re-add-in-block3"), []byte("2")}, - KVPair{[]byte("z-genesis-only"), []byte("2")}, + KVPair{[]byte("add-in-block1"), value1}, + KVPair{[]byte("add-in-block2"), value1}, + KVPair{[]byte("modify-in-block2"), value2}, + KVPair{[]byte("re-add-in-block3"), value2}, + KVPair{[]byte("z-genesis-only"), value2}, }, { - KVPair{[]byte("add-in-block1"), []byte("1")}, - KVPair{[]byte("add-in-block2"), []byte("1")}, - KVPair{[]byte("modify-in-block2"), []byte("2")}, - KVPair{[]byte("z-genesis-only"), []byte("2")}, + KVPair{[]byte("add-in-block1"), value1}, + KVPair{[]byte("add-in-block2"), value1}, + KVPair{[]byte("modify-in-block2"), value2}, + KVPair{[]byte("z-genesis-only"), value2}, }, } for i, exp := range expItems { diff --git a/versiondb/tmdb/store_test.go b/versiondb/tmdb/store_test.go index 702cfe739e..1a4a350c29 100644 --- a/versiondb/tmdb/store_test.go +++ b/versiondb/tmdb/store_test.go @@ -6,7 +6,7 @@ import ( "github.com/cosmos/cosmos-sdk/codec" codectypes "github.com/cosmos/cosmos-sdk/codec/types" - "github.com/crypto-org-chain/cronos/client/file" + "github.com/crypto-org-chain/cronos/client" "github.com/crypto-org-chain/cronos/versiondb" "github.com/crypto-org-chain/cronos/x/cronos/types" "github.com/stretchr/testify/require" @@ -35,7 +35,7 @@ func TestFeed(t *testing.T) { _, err = buf.Write(bz) require.NoError(t, err) } - pairs, err := file.DecodeData(buf.Bytes()) + pairs, err := client.DecodeData(buf.Bytes()) require.NoError(t, err) require.NotEmpty(t, pairs) store := storeCreator() From 97f6f0988c5f15ecba1867f01a29189a4d8a717f Mon Sep 17 00:00:00 2001 From: mmsqe Date: Thu, 27 Oct 2022 09:32:34 +0800 Subject: [PATCH 26/31] fix lint --- client/file/sync.go | 7 +++++-- client/file/watcher.go | 15 +++++++++------ client/file/watcher_test.go | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/client/file/sync.go b/client/file/sync.go index 9de708f489..e947ff8602 100644 --- a/client/file/sync.go +++ b/client/file/sync.go @@ -95,7 +95,10 @@ func Sync(versionDB *tmdb.Store, remoteGrpcUrl, remoteUrl, remoteWsUrl, rootDir pairs, err := client.DecodeData(data.Data) fmt.Printf("mm-pairs: %+v, %+v\n", len(pairs), err) if err == nil { - versionDB.PutAtVersion(int64(data.BlockNum), pairs) + if err = versionDB.PutAtVersion(int64(data.BlockNum), pairs); err != nil { + fmt.Println("mm-put-at-version-panic") + panic(err) + } } case err := <-chErr: // fail read @@ -119,7 +122,7 @@ func Sync(versionDB *tmdb.Store, remoteGrpcUrl, remoteUrl, remoteWsUrl, rootDir case data := <-chData: file := GetLocalDataFileName(directory, data.BlockNum) fmt.Printf("mm-data.BlockNum: %+v\n", data.BlockNum) - if err := os.WriteFile(file, data.Data, 0644); err != nil { + if err := os.WriteFile(file, data.Data, 0600); err != nil { fmt.Println("mm-WriteFile-panic") panic(err) } diff --git a/client/file/watcher.go b/client/file/watcher.go index 739017ea2e..7a170c7475 100644 --- a/client/file/watcher.go +++ b/client/file/watcher.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "net/http" "os" + "path/filepath" "sync" "sync/atomic" "time" @@ -22,7 +23,7 @@ type fileDownloader interface { type localFileDownloader struct{} func (d *localFileDownloader) GetData(path string) ([]byte, error) { - data, err := os.ReadFile(path) + data, err := os.ReadFile(filepath.Clean(path)) if err != nil { if os.IsNotExist(err) { err = errNotExist @@ -114,18 +115,20 @@ func (w *BlockFileWatcher) SetMaxBlockNum(num int) { func (w *BlockFileWatcher) fetch(blockNum int) error { path := w.getFilePath(blockNum) - f, err := os.Open(path) + f, err := os.Open(filepath.Clean(path)) if err == nil { - defer f.Close() //nolint + defer func() { + _ = f.Close() + }() // valid 1st 8 bytes for downloaded file var bytes [8]byte if _, err = io.ReadFull(f, bytes[:]); err == nil { size := binary.BigEndian.Uint64(bytes[:]) - if info, err := f.Stat(); err == nil && size+8 == uint64(info.Size()) { + if info, err := f.Stat(); err == nil && size+uint64(8) == uint64(info.Size()) { return nil } } - f.Close() //nolint + _ = f.Close() } // download if file not exist @@ -203,7 +206,7 @@ func (w *BlockFileWatcher) Start( } } } - for k := range finishedBlockNums { + for k, _ := range finishedBlockNums { if k <= blockNum { delete(finishedBlockNums, k) } diff --git a/client/file/watcher_test.go b/client/file/watcher_test.go index 51d2ad3096..8839516536 100644 --- a/client/file/watcher_test.go +++ b/client/file/watcher_test.go @@ -24,7 +24,7 @@ func setupDirectory(t *testing.T, directory string) func(t *testing.T) { func setupBlockFiles(directory string, start, end int) { for i := start; i <= end; i++ { file := GetLocalDataFileName(directory, i) - os.WriteFile(file, []byte(fmt.Sprint("block", i)), 0644) + os.WriteFile(file, []byte(fmt.Sprint("block", i)), 0600) } } From 8321afc518e7894ecb6f720c7f5d72cd455c4da3 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Tue, 29 Nov 2022 10:48:43 +0800 Subject: [PATCH 27/31] update ethermint before bump geth --- go.mod | 46 ++++----- go.sum | 270 +++++++++---------------------------------------- gomod2nix.toml | 84 +++++++-------- 3 files changed, 113 insertions(+), 287 deletions(-) diff --git a/go.mod b/go.mod index 5ce5150e08..7d903bd389 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,11 @@ require ( cosmossdk.io/math v1.0.0-beta.3 github.com/RoaringBitmap/roaring v1.2.1 github.com/armon/go-metrics v0.4.1 - github.com/cosmos/cosmos-sdk v0.46.3 + github.com/cosmos/cosmos-sdk v0.46.5-0.20221114064055-2114ec42dfa1 github.com/cosmos/gogoproto v1.4.3 - github.com/cosmos/ibc-go/v5 v5.0.0 - github.com/ethereum/go-ethereum v1.10.19 - github.com/evmos/ethermint v0.6.1-0.20221024154336-dca6d45cd75e + github.com/cosmos/ibc-go/v5 v5.1.0 + github.com/ethereum/go-ethereum v1.10.25 + github.com/evmos/ethermint v0.6.1-0.20221116004221-d5917752a66d github.com/gogo/protobuf v1.3.3 github.com/golang/protobuf v1.5.2 github.com/gorilla/mux v1.8.0 @@ -18,12 +18,12 @@ require ( github.com/peggyjv/gravity-bridge/module/v2 v2.0.0-20220420162017-838c0d25e974 github.com/rakyll/statik v0.1.7 github.com/spf13/cast v1.5.0 - github.com/spf13/cobra v1.6.0 + github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.1 github.com/tendermint/tendermint v0.34.24 github.com/tendermint/tm-db v0.6.7 - google.golang.org/genproto v0.0.0-20221018160656-63c7b68cfc55 + google.golang.org/genproto v0.0.0-20221116193143-41c2ba794472 google.golang.org/grpc v1.50.1 google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8 gopkg.in/yaml.v2 v2.4.0 @@ -31,10 +31,11 @@ require ( ) require ( - cloud.google.com/go v0.102.1 // indirect - cloud.google.com/go/compute v1.7.0 // indirect - cloud.google.com/go/iam v0.4.0 // indirect - cloud.google.com/go/storage v1.22.1 // indirect + cloud.google.com/go v0.105.0 // indirect + cloud.google.com/go/compute v1.12.1 // indirect + cloud.google.com/go/compute/metadata v0.2.1 // indirect + cloud.google.com/go/iam v0.7.0 // indirect + cloud.google.com/go/storage v1.27.0 // indirect cosmossdk.io/errors v1.0.0-beta.7 // indirect filippo.io/edwards25519 v1.0.0-rc.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect @@ -61,7 +62,7 @@ require ( github.com/coinbase/rosetta-sdk-go v0.7.9 // indirect github.com/confio/ics23/go v0.7.0 // indirect github.com/cosmos/btcutil v1.0.4 // indirect - github.com/cosmos/cosmos-proto v1.0.0-alpha7 // indirect + github.com/cosmos/cosmos-proto v1.0.0-alpha8 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect github.com/cosmos/gorocksdb v1.2.0 // indirect github.com/cosmos/iavl v0.19.4 // indirect @@ -82,7 +83,7 @@ require ( github.com/dvsekhvalnov/jose2go v1.5.0 // indirect github.com/edsrzf/mmap-go v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.2 // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.1 // indirect @@ -99,9 +100,8 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/orderedcode v0.0.1 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect - github.com/googleapis/gax-go/v2 v2.4.0 // indirect - github.com/googleapis/go-type-adapters v1.0.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/googleapis/gax-go/v2 v2.6.0 // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect @@ -161,7 +161,7 @@ require ( github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/spf13/afero v1.9.2 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/viper v1.13.0 // indirect + github.com/spf13/viper v1.14.0 // indirect github.com/status-im/keycard-go v0.0.0-20200402102358-957c09536969 // indirect github.com/subosito/gotenv v1.4.1 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect @@ -175,16 +175,16 @@ require ( github.com/zondax/hid v0.9.1-0.20220302062450-5552068d2266 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/crypto v0.1.0 // indirect + golang.org/x/crypto v0.3.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect - golang.org/x/net v0.1.0 // indirect - golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2 // indirect - golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 // indirect - golang.org/x/sys v0.1.0 // indirect - golang.org/x/term v0.1.0 // indirect + golang.org/x/net v0.2.0 // indirect + golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.2.0 // indirect + golang.org/x/term v0.2.0 // indirect golang.org/x/text v0.4.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/api v0.93.0 // indirect + google.golang.org/api v0.102.0 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect diff --git a/go.sum b/go.sum index 53237e3370..dc260cceeb 100644 --- a/go.sum +++ b/go.sum @@ -19,21 +19,8 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= -cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= -cloud.google.com/go v0.102.1 h1:vpK6iQWv/2uUeFJth4/cBHsQAGjn1iIE6AAlxipRaA0= -cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= +cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -41,18 +28,15 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= -cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= -cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= -cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= -cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= -cloud.google.com/go/compute v1.7.0 h1:v/k9Eueb8aAJ0vZuxKMrgm6kPhCLZU9HxFU+AFDs9Uk= -cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= -cloud.google.com/go/iam v0.4.0 h1:YBYU00SCDzZJdHqVc4I5d6lsklcYIjQZa1YmEz4jlSE= -cloud.google.com/go/iam v0.4.0/go.mod h1:cbaZxyScUhxl7ZAkNWiALgihfP75wS/fUsVNaa1r3vA= +cloud.google.com/go/iam v0.7.0 h1:k4MuwOsS7zGJJ+QfZ5vBK8SgHBAvYN/23BWsiihJ1vs= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -63,8 +47,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -cloud.google.com/go/storage v1.22.1 h1:F6IlQJZrZM++apn9V5/VfS3gbTUYg98PS3EMQAzqtfg= -cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= +cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= cosmossdk.io/errors v1.0.0-beta.7 h1:gypHW76pTQGVnHKo6QBkb4yFOJjC+sUGRc5Al3Odj1w= cosmossdk.io/errors v1.0.0-beta.7/go.mod h1:mz6FQMJRku4bY7aqS/Gwfcmr/ue91roMEKAmDUDpBfE= @@ -215,12 +199,6 @@ github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3h github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd/v2 v2.0.2 h1:weh8u7Cneje73dDh+2tEVLUvyBc89iwepWCD8b8034E= github.com/cockroachdb/apd/v2 v2.0.2/go.mod h1:DDxRlzC2lo3/vSlmSoS7JkqbbrARPuFOGr0B9pvN3Gw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= @@ -243,8 +221,8 @@ github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cosmos/btcutil v1.0.4 h1:n7C2ngKXo7UC9gNyMNLbzqz7Asuf+7Qv4gnX/rOdQ44= github.com/cosmos/btcutil v1.0.4/go.mod h1:Ffqc8Hn6TJUdDgHBwIZLtrLQC1KdJ9jGJl/TvgUaxbU= -github.com/cosmos/cosmos-proto v1.0.0-alpha7 h1:yqYUOHF2jopwZh4dVQp3xgqwftE5/2hkrwIV6vkUbO0= -github.com/cosmos/cosmos-proto v1.0.0-alpha7/go.mod h1:dosO4pSAbJF8zWCzCoTWP7nNsjcvSUBQmniFxDg5daw= +github.com/cosmos/cosmos-proto v1.0.0-alpha8 h1:d3pCRuMYYvGA5bM0ZbbjKn+AoQD4A7dyNG2wzwWalUw= +github.com/cosmos/cosmos-proto v1.0.0-alpha8/go.mod h1:6/p+Bc4O8JKeZqe0VqUGTX31eoYqemTT4C1hLCWsO7I= github.com/cosmos/cosmos-sdk v0.46.7-0.20221206163158-45d2f08e0e1d h1:Np+fiB2/FUV/lPhFVecebV+nO9qsQgILx/hWcVeKfiI= github.com/cosmos/cosmos-sdk v0.46.7-0.20221206163158-45d2f08e0e1d/go.mod h1:B2j/SQkKvs/hUbXTmKxGwBnShJHvlxtVEbvTYNPwfrU= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= @@ -256,8 +234,8 @@ github.com/cosmos/gorocksdb v1.2.0 h1:d0l3jJG8M4hBouIZq0mDUHZ+zjOx044J3nGRskwTb4 github.com/cosmos/gorocksdb v1.2.0/go.mod h1:aaKvKItm514hKfNJpUJXnnOWeBnk2GL4+Qw9NHizILw= github.com/cosmos/iavl v0.19.4 h1:t82sN+Y0WeqxDLJRSpNd8YFX5URIrT+p8n6oJbJ2Dok= github.com/cosmos/iavl v0.19.4/go.mod h1:X9PKD3J0iFxdmgNLa7b2LYWdsGd90ToV5cAONApkEPw= -github.com/cosmos/ibc-go/v5 v5.0.0 h1:MkObdarpoICPHXoRg/Ne9NRix4j7eQlJZq74/uzH3Zc= -github.com/cosmos/ibc-go/v5 v5.0.0/go.mod h1:Wqsguq98Iuns8tgTv8+xaGYbC+Q8zJfbpjzT6IgMJbs= +github.com/cosmos/ibc-go/v5 v5.1.0 h1:m1NHXFkwwvNeJegZqtyox1WLinh+PMy4ivU/Cs9KjeA= +github.com/cosmos/ibc-go/v5 v5.1.0/go.mod h1:H6sV0/CkNRDtvSrhbsIgiog1WnSwhguGfg8x34MOVEk= github.com/cosmos/ledger-cosmos-go v0.11.1 h1:9JIYsGnXP613pb2vPjFeMMjBI5lEDsEaF6oYorTy6J4= github.com/cosmos/ledger-cosmos-go v0.11.1/go.mod h1:J8//BsAGTo3OC/vDLjMRFLW6q0WAaXvHnVc7ZmE8iUY= github.com/cosmos/ledger-go v0.9.2 h1:Nnao/dLwaVTk1Q5U9THldpUMMXU94BOTWPddSmVB6pI= @@ -329,15 +307,11 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/go-ethereum v1.10.19 h1:EOR5JbL4MD5yeOqv8W2iC1s4NximrTjqFccUz8lyBRA= github.com/ethereum/go-ethereum v1.10.19/go.mod h1:IJBNMtzKcNHPtllYihy6BL2IgK1u+32JriaTbdt4v+w= -github.com/evmos/ethermint v0.6.1-0.20221024154336-dca6d45cd75e h1:2ahOWcXLrVeJsIoBBMj0P/zcvZCK/9QgrR2aJop1rYo= -github.com/evmos/ethermint v0.6.1-0.20221024154336-dca6d45cd75e/go.mod h1:k8w7pACqdY6g9nl/qSguOr1pjfo9JO5g6RIxHaNtLZ8= +github.com/evmos/ethermint v0.6.1-0.20221116004221-d5917752a66d h1:aMxLKbIHOw6MjNSVT0cJS3GZC6Fsc9L+aVKZ0DmVwss= +github.com/evmos/ethermint v0.6.1-0.20221116004221-d5917752a66d/go.mod h1:LFgfNlGiY8NaAweOiFfoP9Qjm5VGBbOGa1rTV+ts5wI= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= @@ -356,8 +330,8 @@ github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2 github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILDlzrGEckF6HKjXe48EgsY/l7K7vhY4MW8= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= @@ -441,9 +415,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -460,7 +432,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -484,9 +455,6 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= @@ -498,7 +466,6 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/orderedcode v0.0.1 h1:UzfcAexk9Vhv8+9pNOgRu41f16lHq725vPwnSeiG/Us= github.com/google/orderedcode v0.0.1/go.mod h1:iVyU4/qPKHY5h/wSd6rZZCDcLJNxiWO6dvsYES2Sb20= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -511,11 +478,6 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -523,19 +485,12 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/enterprise-certificate-proxy v0.1.0 h1:zO8WHNx/MYiAKJ3d5spxZXZE6KHmIQGQcAzwUzV7qQw= -github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= -github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk= -github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= -github.com/googleapis/go-type-adapters v1.0.0 h1:9XdMn+d/G57qq1s8dNc5IesGCXHf6V2HZ2JwRxfA2tA= -github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/googleapis/gax-go/v2 v2.6.0 h1:SXk3ABtQYDT/OH8jAyvEOQ58mgawq5C4o/4/89qN2ZU= +github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= @@ -812,12 +767,12 @@ github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= +github.com/onsi/ginkgo/v2 v2.5.0 h1:TRtrvv2vdQqzkwrQ1ke6vtXf7IK34RBUJafIy1wMwls= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.22.1 h1:pY8O4lBfsHKZHM/6nrxkhVPUznOlIu3quZcKP/M20KI= +github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= @@ -960,8 +915,8 @@ github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= -github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -970,8 +925,8 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.13.0 h1:BWSJ/M+f+3nmdz9bxB+bWX28kkALN2ok11D0rSo8EJU= -github.com/spf13/viper v1.13.0/go.mod h1:Icm2xNL3/8uyh/wFuB1jI7TiTNKp8632Nwegu+zgdYw= +github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= +github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/status-im/keycard-go v0.0.0-20200402102358-957c09536969 h1:Oo2KZNP70KE0+IUJSidPj/BFS/RXNHmKIJOdckzml2E= github.com/status-im/keycard-go v0.0.0-20200402102358-957c09536969/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= @@ -1071,7 +1026,6 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1104,8 +1058,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= +golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1137,7 +1091,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -1199,9 +1152,7 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1209,13 +1160,8 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1225,19 +1171,10 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2 h1:+jnHzr9VPj32ykQVai5DNahi9+NSp7yYuCsl5eAQtL0= -golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1249,9 +1186,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 h1:cu5kTvlzcw1Q5S9f5ip1/cpiB4nXvw1XYzFPGgzLUOY= -golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1314,54 +1250,34 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1379,7 +1295,7 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1439,10 +1355,6 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= @@ -1450,9 +1362,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= @@ -1481,28 +1390,8 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= -google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= -google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= -google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= -google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= -google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= -google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= -google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= -google.golang.org/api v0.93.0 h1:T2xt9gi0gHdxdnRkVQhT8mIvPaXKNsDNWz+L696M66M= -google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.102.0 h1:JxJl2qQ85fRMPNvlZY/enexbxpCjLwGhZUtgfGeQ51I= +google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1555,54 +1444,9 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20221018160656-63c7b68cfc55 h1:U1u4KB2kx6KR/aJDjQ97hZ15wQs8ZPvDcGcRynBhkvg= -google.golang.org/genproto v0.0.0-20221018160656-63c7b68cfc55/go.mod h1:45EK0dUbEZ2NHjCeAd2LXmyjAgGUGrpGROgjhC3ADck= +google.golang.org/genproto v0.0.0-20221116193143-41c2ba794472 h1:kIfItBRE5gkUKpH4H5lNGciZbka1JrmRli3ArqrKFkA= +google.golang.org/genproto v0.0.0-20221116193143-41c2ba794472/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1627,23 +1471,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1656,8 +1485,6 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8 h1:KR8+MyP7/qOlV+8Af01LtjL04bu7on42eVsxT4EyBQk= google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1707,8 +1534,7 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= nhooyr.io/websocket v1.8.6 h1:s+C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k= nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= -pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g= -pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU= +pgregory.net/rapid v0.5.3 h1:163N50IHFqr1phZens4FQOdPgfJscR7a562mjQqeo4M= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/gomod2nix.toml b/gomod2nix.toml index 7f787a5d2c..c19214b596 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -2,17 +2,20 @@ schema = 3 [mod] [mod."cloud.google.com/go"] - version = "v0.102.1" - hash = "sha256-rv3MHFvIrN6AcrmWcWGSQwgaZLbjJMmJFi3bsfHoiO4=" + version = "v0.105.0" + hash = "sha256-2nYtHjuN9ghGcM6aPlOxyNNarHebHtj0Xec48sWwdaI=" [mod."cloud.google.com/go/compute"] - version = "v1.7.0" - hash = "sha256-g+n7L36LC+NP4KaiEu9fCMn6S9fkxjp4PCLOp/oXV38=" + version = "v1.12.1" + hash = "sha256-mMsdL+6R1KuFeAFqEJv8oq/cmRc9hSluEisCpGP0GTc=" + [mod."cloud.google.com/go/compute/metadata"] + version = "v0.2.1" + hash = "sha256-gqjkb6y5POkTWXGQtCvgygvXqoLkESB+cykKp7Mp3jI=" [mod."cloud.google.com/go/iam"] - version = "v0.4.0" - hash = "sha256-sPtj6QPPC/QrkDXJW4Y7rMIHk5l1y9XV/DgM47eEtvg=" + version = "v0.7.0" + hash = "sha256-2UTJ/BbZ+YMGJhyo3HRPUYVDXW6+B5znFdqiutboYtA=" [mod."cloud.google.com/go/storage"] - version = "v1.22.1" - hash = "sha256-YZt05wru0kacyi6moTgvocavActKpXx7FlLh+o0HGDs=" + version = "v1.27.0" + hash = "sha256-V4B6A1Ms8cemB5Cs6nAtUe1N1ldaI9oqTdzGU2FUhrc=" [mod."cosmossdk.io/errors"] version = "v1.0.0-beta.7" hash = "sha256-XblGvIx6Wvvq6wggXjp+KbeJGXoe7AZH7hXEdauCezU=" @@ -101,8 +104,8 @@ schema = 3 version = "v1.0.4" hash = "sha256-JvcBXBdjdmnaW/nyf/tw/uaOAGn1b78yxrtl2/Rs3kA=" [mod."github.com/cosmos/cosmos-proto"] - version = "v1.0.0-alpha7" - hash = "sha256-2wCH+toTF2A6MfFjOa13muEH5oBCcxAhZEqirNOrBA0=" + version = "v1.0.0-alpha8" + hash = "sha256-iXzXoS5Kfh5DBy+PhdFWraDWXda/3M4j7j4VECjv4CA=" [mod."github.com/cosmos/cosmos-sdk"] version = "v0.46.7-0.20221206163158-45d2f08e0e1d" hash = "sha256-4j5nR0VxlxhBeCQS4RvWngQZfZIy2NVNxyTFwmmwGTk=" @@ -120,8 +123,8 @@ schema = 3 version = "v0.19.4" hash = "sha256-EmpRZ48pjPFq/fIHneut9Vyo5QJATfb3ZO7KzWnqs9g=" [mod."github.com/cosmos/ibc-go/v5"] - version = "v5.0.0" - hash = "sha256-sDZdmuGohaaBF7bxrjo9PWJnmoF+VOkjySYhsFixPz4=" + version = "v5.1.0" + hash = "sha256-9ZwGw6XqYkBtj+vTQ2IodNb7VcGesTFnVsxjd+G1I6Q=" [mod."github.com/cosmos/ledger-cosmos-go"] version = "v0.11.1" hash = "sha256-yli+VvVtZmHo2LPvCY6lYVUfcCDn3sBLDL+a8KIlqDA=" @@ -175,14 +178,14 @@ schema = 3 hash = "sha256-7FPnTGcCb8Xd1QVR+6PmGTaHdTY1mm/8osFTW1JLuG8=" replaced = "github.com/ethereum/go-ethereum" [mod."github.com/evmos/ethermint"] - version = "v0.6.1-0.20221024154336-dca6d45cd75e" - hash = "sha256-pDhVKxRe7KKo2/CnyfBMOO4zpy/sPt5fmgU2G4ax5kw=" + version = "v0.6.1-0.20221116004221-d5917752a66d" + hash = "sha256-lHilib77OwhVkZrpmbGVPprtu+lvndeSZpc8zBbOhx4=" [mod."github.com/felixge/httpsnoop"] version = "v1.0.2" hash = "sha256-hj6FZQ1fDAV+1wGIViAt8XaAkWZ1I5vJzgjIJa7XRBA=" [mod."github.com/fsnotify/fsnotify"] - version = "v1.5.4" - hash = "sha256-iPuInpDAJvDeSTr6to713J/fTOWdQIWlLXWAuWIs2Bw=" + version = "v1.6.0" + hash = "sha256-DQesOCweQPEwmAn6s7DCP/Dwy8IypC+osbpfsvpkdP0=" [mod."github.com/gballet/go-libpcsclite"] version = "v0.0.0-20190607065134-2772fd86a8ff" hash = "sha256-Nr5ocU9s1F2Lhx/Zq6/nIo+KkKEqMjDYOEs3yWRC48g=" @@ -239,14 +242,11 @@ schema = 3 version = "v1.3.0" hash = "sha256-QoR55eBtA94T2tBszyxfDtO7/pjZZSGb5vm7U0Xhs0Y=" [mod."github.com/googleapis/enterprise-certificate-proxy"] - version = "v0.1.0" - hash = "sha256-fxaElfiGGh1mLmltkFpVFdiuaagrLZLTW9btVpK13wg=" + version = "v0.2.0" + hash = "sha256-5gxsS5kDLqNFBkQ4eK6GS/sNn+aFknm6jjfteO+6jl4=" [mod."github.com/googleapis/gax-go/v2"] - version = "v2.4.0" - hash = "sha256-zzat4+3iF2XBTQ6RZAUgsCbfK0HgO0nYhM4utA5dqz0=" - [mod."github.com/googleapis/go-type-adapters"] - version = "v1.0.0" - hash = "sha256-u3ajruRV/EN2E1WKet/zoe3zmRrAy4C5F2Dx8bpQwoc=" + version = "v2.6.0" + hash = "sha256-sSsncbJmv0n5jB3G7tHY1Zr2ucCusIeTrXZH4A8QqYI=" [mod."github.com/gorilla/handlers"] version = "v1.5.1" hash = "sha256-GnBAARgOx1E+hDMQ63SI17hdhGtLQxb31lZOmn5j/pU=" @@ -438,8 +438,8 @@ schema = 3 version = "v1.5.0" hash = "sha256-Pdp+wC5FWqyJKzyYHb7JCcV9BoJk/sxQw6nLyuLJvuQ=" [mod."github.com/spf13/cobra"] - version = "v1.6.0" - hash = "sha256-BidkXU9dFuU3Ah8Hl0PbuDe/EHrTr0B1JLSsdFgCXyI=" + version = "v1.6.1" + hash = "sha256-80B5HcYdFisz6QLYkTyka7f9Dr6AfcVyPwp3QChoXwU=" [mod."github.com/spf13/jwalterweatherman"] version = "v1.1.0" hash = "sha256-62BQtqTLF/eVrTOr7pUXE7AiHRjOVC8jQs3/Ehmflfs=" @@ -447,8 +447,8 @@ schema = 3 version = "v1.0.5" hash = "sha256-w9LLYzxxP74WHT4ouBspH/iQZXjuAh2WQCHsuvyEjAw=" [mod."github.com/spf13/viper"] - version = "v1.13.0" - hash = "sha256-yHviS3lMo1hB6jwyiU3zpyGtxP6v9AsyZUKiLeCVz+k=" + version = "v1.14.0" + hash = "sha256-qn3VzD339HZIub5QP4aWVciSufAfVb9llbmQZ8pfDEs=" [mod."github.com/status-im/keycard-go"] version = "v0.0.0-20200402102358-957c09536969" hash = "sha256-yddXXuu6mEFEO2/K6c1tWymeBKzOcvLQnNsFGRjtfXk=" @@ -499,26 +499,26 @@ schema = 3 version = "v0.23.0" hash = "sha256-R3O9GyNtv6j0ic7s+2xkLLaLzbJEop0Otj1nJDFBjsg=" [mod."golang.org/x/crypto"] - version = "v0.1.0" - hash = "sha256-0oZWBSiW5Pd/2a1p2beuoelDe0CpfXZhrg/qPduJlYs=" + version = "v0.3.0" + hash = "sha256-Un9wPqz8u/xpV98T4IqE6RMXIPhGCIm2prsNkHP3cjg=" [mod."golang.org/x/exp"] version = "v0.0.0-20220722155223-a9213eeb770e" hash = "sha256-kNgzydWRpjm0sZl4uXEs3LX5L0xjJtJRAFf/CTlYUN4=" [mod."golang.org/x/net"] - version = "v0.1.0" - hash = "sha256-SrThFBg6sqGYpiN1E3d1SilJxbKkQhSZXPmAFoMAA/I=" + version = "v0.2.0" + hash = "sha256-0MqnHDdLkkau6k7hlWD9MzIoAFROvxulyT+KzZkSXOs=" [mod."golang.org/x/oauth2"] - version = "v0.0.0-20220622183110-fd043fe589d2" - hash = "sha256-VLffpTpx3DlUzXB8mKiJfFzm4ZmgnLSUuLB5Ir0WQUg=" + version = "v0.0.0-20221014153046-6fdb5e3db783" + hash = "sha256-IoygidVNqyAZmN+3macDeIefK8hhJToygpcqlwehdYQ=" [mod."golang.org/x/sync"] - version = "v0.0.0-20220929204114-8fcdb60fdcc0" + version = "v0.1.0" hash = "sha256-Hygjq9euZ0qz6TvHYQwOZEjNiTbTh1nSLRAWZ6KFGR8=" [mod."golang.org/x/sys"] - version = "v0.1.0" - hash = "sha256-nZbEJ/2PuWrDLD4ujeVvcFGoIsfVoIH/Lcp4FjD7hpU=" + version = "v0.2.0" + hash = "sha256-N6yfQH7R2gfcvyWTQZbxWuSNyVy6hAxiab2WFzgAykI=" [mod."golang.org/x/term"] - version = "v0.1.0" - hash = "sha256-UWnNsJIj5nXsuzlPWQ1NyHQuHStaDacMVkFbJ4pnxXk=" + version = "v0.2.0" + hash = "sha256-azcllZ0o/9TurqX9udaJ0o9yxqSoI0/bSJsvQQLYIQc=" [mod."golang.org/x/text"] version = "v0.4.0" hash = "sha256-JvyMygdmTvWg7xhbnUB9MMk6WcYXJt8DAj4DYl82Pys=" @@ -526,14 +526,14 @@ schema = 3 version = "v0.0.0-20220907171357-04be3eba64a2" hash = "sha256-6+zueutgefIYmgXinOflz8qGDDDj0Zhv+2OkGhBTKno=" [mod."google.golang.org/api"] - version = "v0.93.0" - hash = "sha256-W17B79osAhObMbrCHlgywEPg9yIPx0ZISDhp9JwFE5A=" + version = "v0.102.0" + hash = "sha256-q8t08Wu0qzOQOmhY4GSoUrhqLTW/TeXiKhio+5ONfe4=" [mod."google.golang.org/appengine"] version = "v1.6.7" hash = "sha256-zIxGRHiq4QBvRqkrhMGMGCaVL4iM4TtlYpAi/hrivS4=" [mod."google.golang.org/genproto"] - version = "v0.0.0-20221018160656-63c7b68cfc55" - hash = "sha256-Z4ynOw+3NkSkWVOrNaeuQC3G9Q6+joNbTLVoF2HegrU=" + version = "v0.0.0-20221116193143-41c2ba794472" + hash = "sha256-uQuxuOvWRsdMii5M5QresisVd1E+Ss8s2WfR2n7QSXk=" [mod."google.golang.org/grpc"] version = "v1.50.1" hash = "sha256-38nk4qIme+fE57SsCqNxtCZnc8fyzzi4Sb60uDTT2KE=" From ef93aa89a682b9ba77751a37081fff907f5876cf Mon Sep 17 00:00:00 2001 From: mmsqe Date: Wed, 30 Nov 2022 09:14:21 +0800 Subject: [PATCH 28/31] fix resolve --- app/app.go | 26 ++++++++++++++++++++++++++ client/flags.go | 1 - 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/app/app.go b/app/app.go index e6daccc53f..4838b312af 100644 --- a/app/app.go +++ b/app/app.go @@ -6,6 +6,7 @@ import ( "net/http" "os" "path/filepath" + "sync" "github.com/crypto-org-chain/cronos/x/cronos" "github.com/crypto-org-chain/cronos/x/cronos/middleware" @@ -29,6 +30,7 @@ import ( "github.com/cosmos/cosmos-sdk/server/config" servertypes "github.com/cosmos/cosmos-sdk/server/types" "github.com/cosmos/cosmos-sdk/store/streaming" + "github.com/cosmos/cosmos-sdk/store/streaming/file" storetypes "github.com/cosmos/cosmos-sdk/store/types" "github.com/cosmos/cosmos-sdk/testutil/testdata" sdk "github.com/cosmos/cosmos-sdk/types" @@ -150,6 +152,8 @@ const ( // // NOTE: In the SDK, the default value is 255. AddrLen = 20 + + FileStreamerDirectory = "file_streamer" ) // this line is used by starport scaffolding # stargate/wasm/app/enabledProposals @@ -357,6 +361,28 @@ func New( streamers := cast.ToStringSlice(appOpts.Get("store.streamers")) for _, streamerName := range streamers { + if streamerName == "file" { + streamingDir := filepath.Join(cast.ToString(appOpts.Get(flags.FlagHome)), "data", FileStreamerDirectory) + if err := os.MkdirAll(streamingDir, os.ModePerm); err != nil { + panic(err) + } + + // default to exposing all + exposeStoreKeys := make([]storetypes.StoreKey, 0, len(keys)) + for _, storeKey := range keys { + exposeStoreKeys = append(exposeStoreKeys, storeKey) + } + service, err := file.NewStreamingService(streamingDir, "", exposeStoreKeys, appCodec, false, true, true) + if err != nil { + panic(err) + } + bApp.SetStreamingService(service) + + wg := new(sync.WaitGroup) + if err := service.Stream(wg); err != nil { + panic(err) + } + } if streamerName == "versiondb" { rootDir := cast.ToString(appOpts.Get(flags.FlagHome)) dataDir := filepath.Join(rootDir, "data", "versiondb") diff --git a/client/flags.go b/client/flags.go index 4803540414..60a5639404 100644 --- a/client/flags.go +++ b/client/flags.go @@ -1,6 +1,5 @@ package client -const FlagStreamers = "streamers" const FlagIsGrpcOnly = "grpc-only" const FlagIsLocal = "is-local" const FlagLocalDirectory = "local-directory" From 4a9cd2f157de91b5f2e6730da9def05b48fc4075 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Wed, 30 Nov 2022 09:14:36 +0800 Subject: [PATCH 29/31] add onBeforeFetch for streamer --- client/file/sync.go | 33 ++++++++++++++++++++--- client/file/watcher.go | 53 ++++++++++++++----------------------- client/file/watcher_test.go | 4 +-- 3 files changed, 51 insertions(+), 39 deletions(-) diff --git a/client/file/sync.go b/client/file/sync.go index e947ff8602..a5dd0acc5c 100644 --- a/client/file/sync.go +++ b/client/file/sync.go @@ -1,7 +1,9 @@ package file import ( + "encoding/binary" "fmt" + "io" "io/ioutil" "net/http" "os" @@ -85,7 +87,7 @@ func Sync(versionDB *tmdb.Store, remoteGrpcUrl, remoteUrl, remoteWsUrl, rootDir // streamer write the file blk by blk with concurrency 1 streamer := NewBlockFileWatcher(1, maxBlockNum, func(blockNum int) string { return GetLocalDataFileName(directory, blockNum) - }, true) + }, nil, true) streamer.Start(nextBlockNum, interval) go func() { chData, chErr := streamer.SubscribeData(), streamer.SubscribeError() @@ -108,9 +110,32 @@ func Sync(versionDB *tmdb.Store, remoteGrpcUrl, remoteUrl, remoteWsUrl, rootDir } }() - synchronizer := NewBlockFileWatcher(concurrency, maxBlockNum, func(blockNum int) string { - return fmt.Sprintf("%s/%s", remoteUrl, DataFileName(blockNum)) - }, isLocal) + synchronizer := NewBlockFileWatcher( + concurrency, + maxBlockNum, + func(blockNum int) string { + return fmt.Sprintf("%s/%s", remoteUrl, DataFileName(blockNum)) + }, + func(blockNum int) bool { + path := GetLocalDataFileName(directory, blockNum) + f, err := os.Open(path) + if err == nil { + defer func() { + _ = f.Close() + }() + // valid 1st 8 bytes for downloaded file + var bytes [8]byte + if _, err = io.ReadFull(f, bytes[:]); err == nil { + size := binary.BigEndian.Uint64(bytes[:]) + if info, err := f.Stat(); err == nil && size+uint64(8) == uint64(info.Size()) { + return false + } + } + } + return true + }, + isLocal, + ) synchronizer.Start(nextBlockNum, interval) go func() { // max retry for temporary io error diff --git a/client/file/watcher.go b/client/file/watcher.go index 7a170c7475..00526a779c 100644 --- a/client/file/watcher.go +++ b/client/file/watcher.go @@ -1,10 +1,8 @@ package file import ( - "encoding/binary" "errors" "fmt" - "io" "io/ioutil" "net/http" "os" @@ -61,29 +59,32 @@ type BlockData struct { } type BlockFileWatcher struct { - concurrency int - maxBlockNum int64 - getFilePath func(blockNum int) string - downloader fileDownloader - chData chan *BlockData - chError chan error - chDone chan bool - startLock *sync.Mutex + concurrency int + maxBlockNum int64 + getFilePath func(blockNum int) string + onBeforeFetch func(blockNum int) bool + downloader fileDownloader + chData chan *BlockData + chError chan error + chDone chan bool + startLock *sync.Mutex } func NewBlockFileWatcher( concurrency int, maxBlockNum int, getFilePath func(blockNum int) string, + onBeforeFetch func(blockNum int) bool, isLocal bool, ) *BlockFileWatcher { w := &BlockFileWatcher{ - concurrency: concurrency, - maxBlockNum: int64(maxBlockNum), - getFilePath: getFilePath, - chData: make(chan *BlockData), - chError: make(chan error), - startLock: new(sync.Mutex), + concurrency: concurrency, + maxBlockNum: int64(maxBlockNum), + getFilePath: getFilePath, + onBeforeFetch: onBeforeFetch, + chData: make(chan *BlockData), + chError: make(chan error), + startLock: new(sync.Mutex), } if isLocal { w.downloader = new(localFileDownloader) @@ -114,26 +115,12 @@ func (w *BlockFileWatcher) SetMaxBlockNum(num int) { } func (w *BlockFileWatcher) fetch(blockNum int) error { - path := w.getFilePath(blockNum) - f, err := os.Open(filepath.Clean(path)) - if err == nil { - defer func() { - _ = f.Close() - }() - // valid 1st 8 bytes for downloaded file - var bytes [8]byte - if _, err = io.ReadFull(f, bytes[:]); err == nil { - size := binary.BigEndian.Uint64(bytes[:]) - if info, err := f.Stat(); err == nil && size+uint64(8) == uint64(info.Size()) { - return nil - } - } - _ = f.Close() + if w.onBeforeFetch != nil && !w.onBeforeFetch(blockNum) { + return nil } - // download if file not exist + path := w.getFilePath(blockNum) data, err := w.downloader.GetData(path) - fmt.Printf("mm-fetch: %+v, %+v, %+v\n", blockNum, len(data), err) if err != nil { if err != errNotExist { // avoid blocked by error when not subscribe diff --git a/client/file/watcher_test.go b/client/file/watcher_test.go index 8839516536..6bf9d64d98 100644 --- a/client/file/watcher_test.go +++ b/client/file/watcher_test.go @@ -55,7 +55,7 @@ func TestFileWatcher(t *testing.T) { setupBlockFiles(directory, startBlockNum, endBlockNum) watcher := NewBlockFileWatcher(concurrency, endBlockNum+1, func(blockNum int) string { return GetLocalDataFileName(directory, blockNum) - }, true) + }, nil, true) total := start(watcher, startBlockNum, endBlockNum) expected := endBlockNum - startBlockNum + 1 require.Equal(t, expected, total) @@ -71,7 +71,7 @@ func TestFileWatcher(t *testing.T) { }() watcher := NewBlockFileWatcher(concurrency, endBlockNum+1, func(blockNum int) string { return fmt.Sprintf("http://localhost:%s/%s", port, DataFileName(blockNum)) - }, false) + }, nil, false) total := start(watcher, startBlockNum, endBlockNum) expected := endBlockNum - startBlockNum + 1 require.Equal(t, expected, total) From ac15f128af03f625b7245200f145ca00a5e7c14b Mon Sep 17 00:00:00 2001 From: mmsqe Date: Wed, 30 Nov 2022 09:16:56 +0800 Subject: [PATCH 30/31] append len prefix for decode --- client/decoder.go | 2 +- versiondb/tmdb/store_test.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/client/decoder.go b/client/decoder.go index 330ee32d65..f850facdc4 100644 --- a/client/decoder.go +++ b/client/decoder.go @@ -6,7 +6,7 @@ import ( ) func DecodeData(data []byte) (pairs []types.StoreKVPair, err error) { - offset := 0 + offset := 8 for offset < len(data) { size, n := proto.DecodeVarint(data[offset:]) offset += n diff --git a/versiondb/tmdb/store_test.go b/versiondb/tmdb/store_test.go index 1a4a350c29..3ea5a9d660 100644 --- a/versiondb/tmdb/store_test.go +++ b/versiondb/tmdb/store_test.go @@ -6,6 +6,7 @@ import ( "github.com/cosmos/cosmos-sdk/codec" codectypes "github.com/cosmos/cosmos-sdk/codec/types" + sdk "github.com/cosmos/cosmos-sdk/types" "github.com/crypto-org-chain/cronos/client" "github.com/crypto-org-chain/cronos/versiondb" "github.com/crypto-org-chain/cronos/x/cronos/types" @@ -35,7 +36,9 @@ func TestFeed(t *testing.T) { _, err = buf.Write(bz) require.NoError(t, err) } - pairs, err := client.DecodeData(buf.Bytes()) + data := buf.Bytes() + prefix := sdk.Uint64ToBigEndian(uint64(len(data))) + pairs, err := client.DecodeData(append(prefix, data...)) require.NoError(t, err) require.NotEmpty(t, pairs) store := storeCreator() From 575a00396f3224fcf0dfd919d0af715e52a62c11 Mon Sep 17 00:00:00 2001 From: mmsqe Date: Mon, 5 Dec 2022 09:52:36 +0800 Subject: [PATCH 31/31] fail early if incomplete allow retry if incomplete --- client/decoder.go | 13 +++++++++++-- client/file/sync.go | 24 +++++++++++++----------- client/file/watcher.go | 24 ++++++++++++++++++------ 3 files changed, 42 insertions(+), 19 deletions(-) diff --git a/client/decoder.go b/client/decoder.go index f850facdc4..884342fa7c 100644 --- a/client/decoder.go +++ b/client/decoder.go @@ -1,13 +1,22 @@ package client import ( + "fmt" + "github.com/cosmos/cosmos-sdk/store/types" + sdk "github.com/cosmos/cosmos-sdk/types" "github.com/gogo/protobuf/proto" ) func DecodeData(data []byte) (pairs []types.StoreKVPair, err error) { - offset := 8 - for offset < len(data) { + const prefixLen = 8 + offset := prefixLen + dataSize := sdk.BigEndianToUint64(data[:offset]) + size := len(data) + if int(dataSize)+prefixLen != size { + return nil, fmt.Errorf("incomplete file: %v vs %v", dataSize, size) + } + for offset < size { size, n := proto.DecodeVarint(data[offset:]) offset += n pair := new(types.StoreKVPair) diff --git a/client/file/sync.go b/client/file/sync.go index a5dd0acc5c..90845c57ba 100644 --- a/client/file/sync.go +++ b/client/file/sync.go @@ -96,12 +96,13 @@ func Sync(versionDB *tmdb.Store, remoteGrpcUrl, remoteUrl, remoteWsUrl, rootDir case data := <-chData: pairs, err := client.DecodeData(data.Data) fmt.Printf("mm-pairs: %+v, %+v\n", len(pairs), err) - if err == nil { - if err = versionDB.PutAtVersion(int64(data.BlockNum), pairs); err != nil { - fmt.Println("mm-put-at-version-panic") - panic(err) - } + if err != nil { + fmt.Println("invalid decode") + } else if err = versionDB.PutAtVersion(int64(data.BlockNum), pairs); err != nil { + fmt.Println("mm-put-at-version-panic") + panic(err) } + data.ChResult <- err case err := <-chErr: // fail read fmt.Println("mm-fail-read-panic") @@ -150,12 +151,14 @@ func Sync(versionDB *tmdb.Store, remoteGrpcUrl, remoteUrl, remoteWsUrl, rootDir if err := os.WriteFile(file, data.Data, 0600); err != nil { fmt.Println("mm-WriteFile-panic") panic(err) + } else { + retry = 0 + fmt.Println("mm-reset-retry") + if data.BlockNum > maxBlockNum { + streamer.SetMaxBlockNum(data.BlockNum) + } } - retry = 0 - fmt.Println("mm-reset-retry") - if data.BlockNum > maxBlockNum { - streamer.SetMaxBlockNum(data.BlockNum) - } + data.ChResult <- err case err := <-chErr: retry++ fmt.Println("mm-retry", retry) @@ -200,7 +203,6 @@ func Sync(versionDB *tmdb.Store, remoteGrpcUrl, remoteUrl, remoteWsUrl, rootDir blockNum := int(data.Header.Height) fmt.Printf("mm-set-max-blk: %+v\n", blockNum) synchronizer.SetMaxBlockNum(blockNum) - streamer.SetMaxBlockNum(blockNum) } } panic(fmt.Sprintf("max retries %d reached", defaultMaxRetry)) diff --git a/client/file/watcher.go b/client/file/watcher.go index 00526a779c..c1468bf5d1 100644 --- a/client/file/watcher.go +++ b/client/file/watcher.go @@ -8,7 +8,6 @@ import ( "os" "path/filepath" "sync" - "sync/atomic" "time" ) @@ -56,11 +55,12 @@ func (d *httpFileDownloader) GetData(path string) ([]byte, error) { type BlockData struct { BlockNum int Data []byte + ChResult chan<- error } type BlockFileWatcher struct { concurrency int - maxBlockNum int64 + maxBlockNum int getFilePath func(blockNum int) string onBeforeFetch func(blockNum int) bool downloader fileDownloader @@ -68,6 +68,7 @@ type BlockFileWatcher struct { chError chan error chDone chan bool startLock *sync.Mutex + maxBlockLock *sync.RWMutex } func NewBlockFileWatcher( @@ -79,12 +80,13 @@ func NewBlockFileWatcher( ) *BlockFileWatcher { w := &BlockFileWatcher{ concurrency: concurrency, - maxBlockNum: int64(maxBlockNum), + maxBlockNum: maxBlockNum, getFilePath: getFilePath, onBeforeFetch: onBeforeFetch, chData: make(chan *BlockData), chError: make(chan error), startLock: new(sync.Mutex), + maxBlockLock: new(sync.RWMutex), } if isLocal { w.downloader = new(localFileDownloader) @@ -111,7 +113,12 @@ func (w *BlockFileWatcher) SubscribeError() <-chan error { } func (w *BlockFileWatcher) SetMaxBlockNum(num int) { - atomic.StoreInt64(&w.maxBlockNum, int64(num)) + w.maxBlockLock.Lock() + defer w.maxBlockLock.Unlock() + // avoid dup job when set max to smaller one while locking + if num > w.maxBlockNum { + w.maxBlockNum = num + } } func (w *BlockFileWatcher) fetch(blockNum int) error { @@ -131,11 +138,14 @@ func (w *BlockFileWatcher) fetch(blockNum int) error { } return err } + + chResult := make(chan error) w.chData <- &BlockData{ BlockNum: blockNum, Data: data, + ChResult: chResult, } - return nil + return <-chResult } func (w *BlockFileWatcher) Start( @@ -158,7 +168,9 @@ func (w *BlockFileWatcher) Start( default: wg := new(sync.WaitGroup) currentBlockNum := blockNum - maxBlockNum := int(atomic.LoadInt64(&w.maxBlockNum)) + w.maxBlockLock.RLock() + maxBlockNum := w.maxBlockNum + w.maxBlockLock.RUnlock() concurrency := w.concurrency if diff := maxBlockNum - currentBlockNum; diff < concurrency { if diff <= 0 {