From a396d1ad5f80826c75fba042d19ea5659db6bf7b Mon Sep 17 00:00:00 2001 From: emiliocramer Date: Thu, 15 Sep 2022 12:25:36 -0400 Subject: [PATCH 01/11] bring out codec & dmlog -> full.firelog namechange --- {nodemanager/codec => codec}/block_test.go | 0 {nodemanager/codec => codec}/consolereader.go | 10 +- {nodemanager/codec => codec}/init_test.go | 0 {nodemanager/codec => codec}/logging.go | 0 codec/testdata/full.firelog | 1 + .../testdata/full.firelog.golden.json | 0 nodemanager/codec/consolereader_test.go | 152 ------------------ nodemanager/codec/testdata/deep-mind.dmlog | 1 - 8 files changed, 6 insertions(+), 158 deletions(-) rename {nodemanager/codec => codec}/block_test.go (100%) rename {nodemanager/codec => codec}/consolereader.go (92%) rename {nodemanager/codec => codec}/init_test.go (100%) rename {nodemanager/codec => codec}/logging.go (100%) create mode 100644 codec/testdata/full.firelog rename nodemanager/codec/testdata/deep-mind.dmlog.golden.json => codec/testdata/full.firelog.golden.json (100%) delete mode 100644 nodemanager/codec/consolereader_test.go delete mode 100644 nodemanager/codec/testdata/deep-mind.dmlog diff --git a/nodemanager/codec/block_test.go b/codec/block_test.go similarity index 100% rename from nodemanager/codec/block_test.go rename to codec/block_test.go diff --git a/nodemanager/codec/consolereader.go b/codec/consolereader.go similarity index 92% rename from nodemanager/codec/consolereader.go rename to codec/consolereader.go index 9eb55ea..781688e 100644 --- a/nodemanager/codec/consolereader.go +++ b/codec/consolereader.go @@ -61,7 +61,7 @@ func newParsingStats(block uint64) *parsingStats { } func (s *parsingStats) log() { - zlog.Info("mindreader block stats", + zlog.Info("reader block stats", zap.Uint64("block_num", s.blockNum), zap.Int64("duration", int64(time.Since(s.startAt))), zap.Reflect("stats", s.data), @@ -83,7 +83,7 @@ func (r *ConsoleReader) ReadBlock() (out *bstream.Block, err error) { } const ( - LogPrefix = "DMLOG" + LogPrefix = "FIRE" LogBlock = "BLOCK" ) @@ -113,7 +113,7 @@ func (r *ConsoleReader) next() (out *bstream.Block, err error) { default: if tracer.Enabled() { - zlog.Debug("skipping unknown deep mind log line", zap.String("line", line)) + zlog.Debug("skipping unknown Firehose log line", zap.String("line", line)) } continue } @@ -147,7 +147,7 @@ func (r *ConsoleReader) buildScanner(reader io.Reader) *bufio.Scanner { } // Format: -// DMLOG BLOCK +// FIRE BLOCK func (r *ConsoleReader) readBlock(params []string) (*pbarweave.Block, error) { if err := validateChunk(params, 2); err != nil { return nil, fmt.Errorf("invalid log line length: %w", err) @@ -177,7 +177,7 @@ func (r *ConsoleReader) readBlock(params []string) (*pbarweave.Block, error) { } if blockHeight != block.Height { - return nil, fmt.Errorf("block height %d from 'DMLOG ...' does not match height %d from block's content", blockHeight, block.Height) + return nil, fmt.Errorf("block height %d from 'FIRE ...' does not match height %d from block's content", blockHeight, block.Height) } if tracer.Enabled() { diff --git a/nodemanager/codec/init_test.go b/codec/init_test.go similarity index 100% rename from nodemanager/codec/init_test.go rename to codec/init_test.go diff --git a/nodemanager/codec/logging.go b/codec/logging.go similarity index 100% rename from nodemanager/codec/logging.go rename to codec/logging.go diff --git a/codec/testdata/full.firelog b/codec/testdata/full.firelog new file mode 100644 index 0000000..4ee232d --- /dev/null +++ b/codec/testdata/full.firelog @@ -0,0 +1 @@ +FIRE BLOCK 1 0801123021e216f6971796c16692808d502d1d4994104bfbc43b934120a658e3fcab0fccac0095a13093d0cb7e16f73154ed92001ab60100000000000101000101000101010101010100000101000100000100000000010001000100000000000001000001000101000101000000010101000100010100010001000000000100010001010100000000010101000000010001000100010100000101000001000000010101000001010101000100000101010100000100000100010001000101000101000101010001000000000001010001000000000001010101010000000101000000000100000000010100012230ef0214ecaa252020230a5325719dfc2d9cec86123bc46926dad0c2251ed6be17b7112528dbe678fb2d31d6e6a09512442a0608cedcebd805320608cddcebd8053a02323840014a3000000007c12435f7e82ef3f3da58f244886a05aee0cecff462c1e527988e0b7aad133779357c8084ecfff5c8c36a575a52006220dad7b9f1d54610a2fdb8cce5336cf6b837e823b6499a21b5b5edbfa3eb97ab686a2b6950654b3447306b5f394f6574637a30784131677466754c51736d4b4d35783530705175626465397877417a0130820101308a010130aa01030a0131 \ No newline at end of file diff --git a/nodemanager/codec/testdata/deep-mind.dmlog.golden.json b/codec/testdata/full.firelog.golden.json similarity index 100% rename from nodemanager/codec/testdata/deep-mind.dmlog.golden.json rename to codec/testdata/full.firelog.golden.json diff --git a/nodemanager/codec/consolereader_test.go b/nodemanager/codec/consolereader_test.go deleted file mode 100644 index b0a74f6..0000000 --- a/nodemanager/codec/consolereader_test.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2021 dfuse Platform Inc. // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package codec - -import ( - "bytes" - "encoding/json" - "io" - "io/ioutil" - "os" - "os/exec" - "reflect" - "strings" - "testing" - - pbarweave "github.com/streamingfast/firehose-arweave/types/pb/sf/arweave/type/v1" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" -) - -type ObjectReader func() (interface{}, error) - -func TestParseFromFile(t *testing.T) { - zlog, _ = zap.NewDevelopment() - tests := []struct { - deepMindFile string - expectedPanicErr error - }{ - // Skipping as the data is broken - // {"testdata/deep-mind.dmlog", nil}, - } - - for _, test := range tests { - t.Run(strings.Replace(test.deepMindFile, "testdata/", "", 1), func(t *testing.T) { - defer func() { - if r := recover(); r != nil { - require.Equal(t, test.expectedPanicErr, r) - } - }() - - cr := testFileConsoleReader(t, test.deepMindFile) - buf := &bytes.Buffer{} - buf.Write([]byte("[")) - - for first := true; true; first = false { - out, err := cr.ReadBlock() - if err == io.EOF { - break - } - require.NoError(t, err) - - block := out.ToProtocol().(*pbarweave.Block) - - if !isNil(block) { - if !first { - buf.Write([]byte(",")) - } - - // FIXMME: jsonpb needs to be updated to latest version of used gRPC - // elements. We are disaligned and using that breaks now. - // Needs to check what is the latest way to properly serialize - // Proto generated struct to JSON. - // value, err := jsonpb.MarshalIndentToString(v, " ") - // require.NoError(t, err) - - value, err := json.MarshalIndent(block, "", " ") - require.NoError(t, err) - - buf.Write(value) - } - - if len(buf.Bytes()) != 0 { - buf.Write([]byte("\n")) - } - - } - buf.Write([]byte("]")) - - goldenFile := test.deepMindFile + ".golden.json" - if os.Getenv("GOLDEN_UPDATE") == "true" { - ioutil.WriteFile(goldenFile, buf.Bytes(), os.ModePerm) - } - - cnt, err := ioutil.ReadFile(goldenFile) - require.NoError(t, err) - - if !assert.Equal(t, string(cnt), buf.String()) { - t.Error("previous diff:\n" + unifiedDiff(t, cnt, buf.Bytes())) - } - }) - } -} - -func isNil(v interface{}) bool { - if v == nil { - return true - } - - rv := reflect.ValueOf(v) - return rv.Kind() == reflect.Ptr && rv.IsNil() -} - -func testFileConsoleReader(t *testing.T, filename string) *ConsoleReader { - t.Helper() - - fl, err := os.Open(filename) - require.NoError(t, err) - - cr := testReaderConsoleReader(t, make(chan string, 10000), func() { fl.Close() }) - - go cr.ProcessData(fl) - - return cr -} - -func testReaderConsoleReader(t *testing.T, lines chan string, closer func()) *ConsoleReader { - t.Helper() - - l := &ConsoleReader{ - lines: lines, - close: closer, - } - - return l -} - -func unifiedDiff(t *testing.T, cnt1, cnt2 []byte) string { - file1 := "/tmp/gotests-linediff-1" - file2 := "/tmp/gotests-linediff-2" - err := ioutil.WriteFile(file1, cnt1, 0600) - require.NoError(t, err) - - err = ioutil.WriteFile(file2, cnt2, 0600) - require.NoError(t, err) - - cmd := exec.Command("diff", "-u", file1, file2) - out, _ := cmd.Output() - - return string(out) -} diff --git a/nodemanager/codec/testdata/deep-mind.dmlog b/nodemanager/codec/testdata/deep-mind.dmlog deleted file mode 100644 index 29b149c..0000000 --- a/nodemanager/codec/testdata/deep-mind.dmlog +++ /dev/null @@ -1 +0,0 @@ -DMLOG BLOCK 1 0801123021e216f6971796c16692808d502d1d4994104bfbc43b934120a658e3fcab0fccac0095a13093d0cb7e16f73154ed92001ab60100000000000101000101000101010101010100000101000100000100000000010001000100000000000001000001000101000101000000010101000100010100010001000000000100010001010100000000010101000000010001000100010100000101000001000000010101000001010101000100000101010100000100000100010001000101000101000101010001000000000001010001000000000001010101010000000101000000000100000000010100012230ef0214ecaa252020230a5325719dfc2d9cec86123bc46926dad0c2251ed6be17b7112528dbe678fb2d31d6e6a09512442a0608cedcebd805320608cddcebd8053a02323840014a3000000007c12435f7e82ef3f3da58f244886a05aee0cecff462c1e527988e0b7aad133779357c8084ecfff5c8c36a575a52006220dad7b9f1d54610a2fdb8cce5336cf6b837e823b6499a21b5b5edbfa3eb97ab686a2b6950654b3447306b5f394f6574637a30784131677466754c51736d4b4d35783530705175626465397877417a0130820101308a010130aa01030a0131 \ No newline at end of file From 61ef6335b6428b99a03cc900f7d054d22d086e46 Mon Sep 17 00:00:00 2001 From: emiliocramer Date: Thu, 15 Sep 2022 12:28:08 -0400 Subject: [PATCH 02/11] mindreader-node.go -> reader-node.go --- .../{mindreader-node.go => reader-node.go} | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) rename cmd/firearweave/cli/{mindreader-node.go => reader-node.go} (51%) diff --git a/cmd/firearweave/cli/mindreader-node.go b/cmd/firearweave/cli/reader-node.go similarity index 51% rename from cmd/firearweave/cli/mindreader-node.go rename to cmd/firearweave/cli/reader-node.go index 8ecf3f2..48eddf7 100644 --- a/cmd/firearweave/cli/mindreader-node.go +++ b/cmd/firearweave/cli/reader-node.go @@ -15,12 +15,12 @@ package cli import ( + "github.com/streamingfast/firehose-arweave/codec" "time" "github.com/spf13/cobra" "github.com/streamingfast/bstream" "github.com/streamingfast/bstream/blockstream" - "github.com/streamingfast/firehose-arweave/nodemanager/codec" pbarweave "github.com/streamingfast/firehose-arweave/types/pb/sf/arweave/type/v1" "github.com/streamingfast/logging" nodeManager "github.com/streamingfast/node-manager" @@ -29,28 +29,28 @@ import ( ) func init() { - registerNode("mindreader", registerMindreaderNodeFlags, MindreaderNodeManagerAPIAddr) + registerNode("reader", registerReaderNodeFlags, ReaderNodeManagerAPIAddr) } -func registerMindreaderNodeFlags(cmd *cobra.Command) error { - cmd.Flags().String("mindreader-node-grpc-listen-addr", MindreaderGRPCAddr, "gRPC listening address to use for serving real-time blocks") - cmd.Flags().Bool("mindreader-node-discard-after-stop-num", false, "Ignore remaining blocks being processed after stop num (only useful if we discard the mindreader data after reprocessing a chunk of blocks)") - cmd.Flags().String("mindreader-node-working-dir", "{data-dir}/mindreader/work", "Path where mindreader will stores its files") - cmd.Flags().Uint("mindreader-node-start-block-num", 0, "Blocks that were produced with smaller block number then the given block num are skipped") - cmd.Flags().Uint("mindreader-node-stop-block-num", 0, "Shutdown mindreader when we the following 'stop-block-num' has been reached, inclusively.") - cmd.Flags().Int("mindreader-node-blocks-chan-capacity", 100, "Capacity of the channel holding blocks read by the mindreader. Process will shutdown superviser/geth if the channel gets over 90% of that capacity to prevent horrible consequences. Raise this number when processing tiny blocks very quickly") - cmd.Flags().String("mindreader-node-one-block-suffix", "default", FlagDescription(` - Unique identifier for Mindreader, so that it can produce 'oneblock files' in the same store as another instance without competing - for writes. You should set this flag if you have multiple mindreader running, each one should get a unique identifier, the +func registerReaderNodeFlags(cmd *cobra.Command) error { + cmd.Flags().String("reader-node-grpc-listen-addr", ReaderGRPCAddr, "gRPC listening address to use for serving real-time blocks") + cmd.Flags().Bool("reader-node-discard-after-stop-num", false, "Ignore remaining blocks being processed after stop num (only useful if we discard the reader data after reprocessing a chunk of blocks)") + cmd.Flags().String("reader-node-working-dir", "{data-dir}/reader/work", "Path where reader will stores its files") + cmd.Flags().Uint("reader-node-start-block-num", 0, "Blocks that were produced with smaller block number then the given block num are skipped") + cmd.Flags().Uint("reader-node-stop-block-num", 0, "Shutdown reader when we the following 'stop-block-num' has been reached, inclusively.") + cmd.Flags().Int("reader-node-blocks-chan-capacity", 100, "Capacity of the channel holding blocks read by the reader. Process will shutdown superviser/geth if the channel gets over 90% of that capacity to prevent horrible consequences. Raise this number when processing tiny blocks very quickly") + cmd.Flags().String("reader-node-one-block-suffix", "default", FlagDescription(` + Unique identifier for Reader, so that it can produce 'oneblock files' in the same store as another instance without competing + for writes. You should set this flag if you have multiple reader running, each one should get a unique identifier, the hostname value is a good value to use. `)) - cmd.Flags().Duration("mindreader-node-wait-upload-complete-on-shutdown", 30*time.Second, "When the mindreader is shutting down, it will wait up to that amount of time for the archiver to finish uploading the blocks before leaving anyway") - cmd.Flags().String("mindreader-node-merge-threshold-block-age", "24h", "When processing blocks with a blocktime older than this threshold, they will be automatically merged") + cmd.Flags().Duration("reader-node-wait-upload-complete-on-shutdown", 30*time.Second, "When the reader is shutting down, it will wait up to that amount of time for the archiver to finish uploading the blocks before leaving anyway") + cmd.Flags().String("reader-node-merge-threshold-block-age", "24h", "When processing blocks with a blocktime older than this threshold, they will be automatically merged") return nil } -func getMindreaderLogPlugin( +func getReaderLogPlugin( blockStreamServer *blockstream.Server, oneBlockStoreURL string, mergedBlockStoreURL string, From de23050ca5e90e46f359e1e82a7e250242356f2e Mon Sep 17 00:00:00 2001 From: emiliocramer Date: Thu, 15 Sep 2022 12:30:18 -0400 Subject: [PATCH 03/11] mindreader -> reader, deepmind -> firehose --- INTEGRATION.md | 38 ++--- README.md | 6 +- cmd/firearweave/cli/constants.go | 16 +- cmd/firearweave/cli/firehose.go | 2 +- cmd/firearweave/cli/main.go | 2 +- cmd/firearweave/cli/node.go | 58 +++---- cmd/firearweave/cli/relayer.go | 2 +- codec/consolereader_test.go | 152 ++++++++++++++++++ devel/live-blocks/live-blocks.yaml | 6 +- devel/reproc-mainnet/reproc-mainnet.yaml | 10 +- devel/standard/standard.yaml | 10 +- nodemanager/logging.go | 8 +- nodemanager/superviser.go | 16 +- tools/fireacme/99-firehose-acme.sh | 2 +- tools/fireacme/motd_node_manager | 2 +- ...-mind-30s => acme-debug-firehose-logs-30s} | 6 +- 16 files changed, 244 insertions(+), 92 deletions(-) create mode 100644 codec/consolereader_test.go rename tools/fireacme/scripts/{acme-debug-deep-mind-30s => acme-debug-firehose-logs-30s} (58%) diff --git a/INTEGRATION.md b/INTEGRATION.md index ae0f222..ceca802 100644 --- a/INTEGRATION.md +++ b/INTEGRATION.md @@ -1,9 +1,9 @@ # Chain Integration Document ## Concepts -Blockchain data extraction occurs by two processes running conjunction a `Deepmind` & `Mindreader`. We run an instrumented version of a process (usually a node) to sync the chain referred to as `DeepMind`. -The DeepMind process instruments the blockchain and outputs logs over the standard output pipe, which is subsequently read and processed by the `Mindreader` process. -The MindReader process will read, and stitch together the output of `DeepMind` to create rich blockchain data models, which it will subsequently write to +Blockchain data extraction occurs by two processes running conjunction a `Firehose` & `Reader`. We run an instrumented version of a process (usually a node) to sync the chain referred to as `Firehose`. +The Firehose process instruments the blockchain and outputs logs over the standard output pipe, which is subsequently read and processed by the `Reader` process. +The Reader process will read, and stitch together the output of `Firehose` to create rich blockchain data models, which it will subsequently write to files. The data models in question are [Google Protobuf Structures](https://developers.google.com/protocol-buffers). #### Data Modeling @@ -20,14 +20,14 @@ https://github.com/streamingfast/proto-ethereum/blob/develop/sf/ethereum/codec/v We have built an end-to-end template, to start the on-boarding process of new chains. This solution consist of: *firehose-acme* -As mentioned above, the `Mindreader` process consumes the data that is extracted and streamed from `Deeepmind`. In Actuality the MindReader +As mentioned above, the `Reader` process consumes the data that is extracted and streamed from `Deepmind`. In Actuality the Reader is one process out of multiple ones that creates the _Firehose_. These processes are launched by one application. This application is chain specific and by convention, we name is "firehose-". Though this application is chain specific, the structure of the application is standardized and is quite similar from chain to chain. For convenience, we have create a boiler plate app to help you get started. We named our chain `Acme` this the app is [firehose-acme](https://github.com/streamingfast/firehose-acme) -*DeepMind* -`Deepmind` consist of an instrumented syncing node. We have created a "dummy" chain to simulate a node process syncing that can be found [https://github.com/streamingfast/dummy-blockchain](https://github.com/streamingfast/dummy-blockchain). +*Firehose* +`Firehose` consist of an instrumented syncing node. We have created a "dummy" chain to simulate a node process syncing that can be found [https://github.com/streamingfast/dummy-blockchain](https://github.com/streamingfast/dummy-blockchain). ## Setting up the dummy chain @@ -69,7 +69,7 @@ cd devel/standard/ vi standard.yaml ``` -modify the flag `mindreader-node-path: "dchain"` to point to the path of your `dchain` binary you compiled above +modify the flag `reader-node-path: "dchain"` to point to the path of your `dchain` binary you compiled above ## Starting and testing Firehose @@ -80,22 +80,22 @@ Start `fireacme` ./start.sh ``` -This will launch `fireacme` application. Behind the scenes we are starting 3 sub processes: `mindreader-node`, `relayer`, `firehose` +This will launch `fireacme` application. Behind the scenes we are starting 3 sub processes: `reader-node`, `relayer`, `firehose` -*mindreader-node* +*reader-node* -The mindreader-node is a process that runs and manages the blockchain node Geth. It consumes the blockchain data that is -extracted from our instrumented Geth node. The instrumented Geth node outputs individual block data. The mindreader-node +The reader-node is a process that runs and manages the blockchain node Geth. It consumes the blockchain data that is +extracted from our instrumented Geth node. The instrumented Geth node outputs individual block data. The reader-node process will either write individual block data into separate files called one-block files or merge 100 blocks data together and write into a file called 100-block file. -This behaviour is configurable with the mindreader-node-merge-and-store-directly flag. When running the mindreader-node -process with mindreader-node-merge-and-store-directly flag enable, we say the “mindreader is running in merged mode”. -When the flag is disabled, we will refer to the mindreader as running in its normal mode of operation. +This behaviour is configurable with the reader-node-merge-and-store-directly flag. When running the reader-node +process with reader-node-merge-and-store-directly flag enable, we say the “reader is running in merged mode”. +When the flag is disabled, we will refer to the reader as running in its normal mode of operation. -In the scenario where the mindreader-node process stores one-block files. We can run a merger process on the side which -would merge the one-block files into 100-block files. When we are syncing the chain we will run the mindreader-node process -in merged mode. When we are synced we will run the mindreader-node in it’s regular mode of operation (storing one-block files) +In the scenario where the reader-node process stores one-block files. We can run a merger process on the side which +would merge the one-block files into 100-block files. When we are syncing the chain we will run the reader-node process +in merged mode. When we are synced we will run the reader-node in it’s regular mode of operation (storing one-block files) The one-block files and 100-block files will be store in data-dir/storage/merged-blocks and data-dir/storage/one-blocks respectively. The naming convention of the file is the number of the first block in the file. @@ -111,8 +111,8 @@ We have also built tools that allow you to introspect block files: go install ../../cmd/fireacme && fireacme tools print blocks --store ./fire-data/storage/merged-blocks 100 ``` -At this point we have `mindreader-node` process running as well a `relayer` & `firehose` process. Both of these processes work together to provide the Firehose data stream. -Once the firehose process is running, it will be listening on port 13042. At it’s core the firehose is a gRPC stream. We can list the available gRPC service +At this point we have `reader-node` process running as well a `relayer` & `firehose` process. Both of these processes work together to provide the Firehose data stream. +Once the Firehose process is running, it will be listening on port 13042. At it’s core the Firehose is a gRPC stream. We can list the available gRPC service ```bash grpcurl -plaintext localhost:16042 list diff --git a/README.md b/README.md index f8dcc0c..17fdb26 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,9 @@ of Arweave blocks. ``` # Arguments shared with both thegarii: # -# --mindreader-node-start-block-num -# --mindreader-node-stop-block-num -./devel/standard/start.sh -- --mindreader-node-start-block-num 911988 +# --reader-node-start-block-num +# --reader-node-stop-block-num +./devel/standard/start.sh -- --reader-node-start-block-num 911988 ``` ## Release diff --git a/cmd/firearweave/cli/constants.go b/cmd/firearweave/cli/constants.go index d5f71e0..f0fc25f 100644 --- a/cmd/firearweave/cli/constants.go +++ b/cmd/firearweave/cli/constants.go @@ -10,12 +10,12 @@ const ( // The current prefix is the one you should use for your chain. Once you have copied // this whole repository, you should open a PR on firehose-arweave to bump it again // so the next team supporting Firehose will use 16XXXX and so forth. - MindreaderGRPCAddr string = ":16010" - NodeManagerAPIAddr string = ":16041" - MindreaderNodeManagerAPIAddr string = ":16009" - MergerServingAddr string = ":16012" - RelayerServingAddr string = ":16011" - FirehoseGRPCServingAddr string = ":16042" + ReaderGRPCAddr string = ":16010" + NodeManagerAPIAddr string = ":16041" + ReaderNodeManagerAPIAddr string = ":16009" + MergerServingAddr string = ":16012" + RelayerServingAddr string = ":16011" + FirehoseGRPCServingAddr string = ":16042" // Data storage default locations ATMDirectory string = "file://{data-dir}/atm" @@ -29,8 +29,8 @@ const ( // Native node instance port definitions, adjust those for your chain // usually all chains have a P2P and RPC port available - MindreaderNodeP2PPort string = "30305" - MindreaderNodeRPCPort string = "8547" + ReaderNodeP2PPort string = "30305" + ReaderNodeRPCPort string = "8547" NodeP2PPort string = "30303" NodeRPCPort string = "3030" diff --git a/cmd/firearweave/cli/firehose.go b/cmd/firearweave/cli/firehose.go index e817877..e4579ae 100644 --- a/cmd/firearweave/cli/firehose.go +++ b/cmd/firearweave/cli/firehose.go @@ -26,7 +26,7 @@ func init() { Title: "Block Firehose", Description: "Provides on-demand filtered blocks, depends on mergd blocks and live source", RegisterFlags: func(cmd *cobra.Command) error { - cmd.Flags().String("firehose-grpc-listen-addr", FirehoseGRPCServingAddr, "Address on which the firehose will listen, appending * to the end of the listen address will start the server over an insecure TLS connection. By default firehose will start in plain-text mode.") + cmd.Flags().String("firehose-grpc-listen-addr", FirehoseGRPCServingAddr, "Address on which the firehose will listen, appending * to the end of the listen address will start the server over an insecure TLS connection. By default Firehose will start in plain-text mode.") return nil }, diff --git a/cmd/firearweave/cli/main.go b/cmd/firearweave/cli/main.go index da91c2b..19e1ad8 100644 --- a/cmd/firearweave/cli/main.go +++ b/cmd/firearweave/cli/main.go @@ -66,7 +66,7 @@ func Main() { derr.Check("arweave-blockchain", RootCmd.Execute()) } -var startCmdExample = `firearweave start mindreader` +var startCmdExample = `firearweave start reader` var startCmdHelpTemplate = `Usage:{{if .Runnable}} {{.UseLine}}{{end}} [all|command1 [command2...]]{{if gt (len .Aliases) 0}} diff --git a/cmd/firearweave/cli/node.go b/cmd/firearweave/cli/node.go index c1f4c53..f6acdbb 100644 --- a/cmd/firearweave/cli/node.go +++ b/cmd/firearweave/cli/node.go @@ -26,16 +26,16 @@ import ( var nodeLogger, nodeTracer = logging.PackageLogger("node", "github.com/streamingfast/firehose-arweave/node") var nodeArweaveLogger, nodeArweaveTracer = logging.PackageLogger("node.arweave", "github.com/streamingfast/firehose-arweave/node/dummy-chain", DefaultLevelInfo) -var mindreaderLogger, mindreaderTracer = logging.PackageLogger("mindreader", "github.com/streamingfast/firehose-arweave/mindreader") -var mindreaderArweaveLogger, mindreaderArweaveTracer = logging.PackageLogger("mindreader.arweave", "github.com/streamingfast/firehose-arweave/mindreader/dummy-chain", DefaultLevelInfo) +var readerLogger, readerTracer = logging.PackageLogger("reader", "github.com/streamingfast/firehose-arweave/reader") +var readerArweaveLogger, readerArweaveTracer = logging.PackageLogger("reader.arweave", "github.com/streamingfast/firehose-arweave/reader/dummy-chain", DefaultLevelInfo) func registerCommonNodeFlags(cmd *cobra.Command, flagPrefix string, managerAPIAddr string) { cmd.Flags().String(flagPrefix+"path", "thegarii", FlagDescription(` - Process that will be invoked mindreader (a.k.a extractor) component, can be a full path or just the binary's name, in which case the binary is + Process that will be invoked reader (a.k.a reader) component, can be a full path or just the binary's name, in which case the binary is searched for paths listed by the PATH environment variable (following operating system rules around PATH handling). `)) - cmd.Flags().String(flagPrefix+"data-dir", "{data-dir}/{node-role}/data", "Directory for node data ({node-role} is either mindreader, peering or dev-miner)") - cmd.Flags().Bool(flagPrefix+"debug-deep-mind", false, "[DEV] Prints deep mind instrumentation logs to standard output, should be use for debugging purposes only") + cmd.Flags().String(flagPrefix+"data-dir", "{data-dir}/{node-role}/data", "Directory for node data ({node-role} is either reader, peering or dev-miner)") + cmd.Flags().Bool(flagPrefix+"debug-firehose-logs", false, "[DEV] Prints Firehose instrumentation logs to standard output, should be use for debugging purposes only") cmd.Flags().Bool(flagPrefix+"log-to-zap", true, FlagDescription(` When sets to 'true', all standard error output emitted by the invoked process defined via '%s' is intercepted, split line by line and each line is then transformed and logged through the Firehose stack @@ -49,8 +49,8 @@ func registerCommonNodeFlags(cmd *cobra.Command, flagPrefix string, managerAPIAd } func registerNode(kind string, extraFlagRegistration func(cmd *cobra.Command) error, managerAPIaddr string) { - if kind != "mindreader" { - panic(fmt.Errorf("invalid kind value, must be either 'mindreader', got %q", kind)) + if kind != "reader" { + panic(fmt.Errorf("invalid kind value, must be either 'reader', got %q", kind)) } app := fmt.Sprintf("%s-node", kind) @@ -83,11 +83,11 @@ func nodeFactoryFunc(flagPrefix, kind string) func(*launcher.Runtime) (launcher. appLogger = nodeLogger appTracer = nodeTracer supervisedProcessLogger = nodeArweaveLogger - case "mindreader": - appLogger = mindreaderLogger - appTracer = mindreaderTracer + case "reader": + appLogger = readerLogger + appTracer = readerTracer - supervisedProcessLogger = mindreaderArweaveLogger + supervisedProcessLogger = readerArweaveLogger default: panic(fmt.Errorf("unknown node kind %q", kind)) } @@ -98,13 +98,13 @@ func nodeFactoryFunc(flagPrefix, kind string) func(*launcher.Runtime) (launcher. nodeDataDir := replaceNodeRole(kind, mustReplaceDataDir(sfDataDir, viper.GetString(flagPrefix+"data-dir"))) readinessMaxLatency := viper.GetDuration(flagPrefix + "readiness-max-latency") - debugDeepMind := viper.GetBool(flagPrefix + "debug-deep-mind") + debugFirehoseLogs := viper.GetBool(flagPrefix + "debug-firehose-logs") logToZap := viper.GetBool(flagPrefix + "log-to-zap") shutdownDelay := viper.GetDuration("common-system-shutdown-signal-delay") // we reuse this global value httpAddr := viper.GetString(flagPrefix + "manager-api-addr") - batchStartBlockNum := viper.GetUint64("mindreader-node-start-block-num") - batchStopBlockNum := viper.GetUint64("mindreader-node-stop-block-num") - endpoints := viper.GetStringSlice("mindreader-node-endpoints") + batchStartBlockNum := viper.GetUint64("reader-node-start-block-num") + batchStopBlockNum := viper.GetUint64("reader-node-stop-block-num") + endpoints := viper.GetStringSlice("reader-node-endpoints") arguments := viper.GetString(flagPrefix + "arguments") nodeArguments, err := buildNodeArguments( @@ -125,7 +125,7 @@ func nodeFactoryFunc(flagPrefix, kind string) func(*launcher.Runtime) (launcher. nodeArguments, nodeDataDir, metricsAndReadinessManager.UpdateHeadBlock, - debugDeepMind, + debugFirehoseLogs, logToZap, appLogger, supervisedProcessLogger, @@ -148,7 +148,7 @@ func nodeFactoryFunc(flagPrefix, kind string) func(*launcher.Runtime) (launcher. return nil, fmt.Errorf("unable to create chain operator: %w", err) } - if kind != "mindreader" { + if kind != "reader" { return nodeManagerApp.New(&nodeManagerApp.Config{ HTTPAddr: httpAddr, }, &nodeManagerApp.Modules{ @@ -160,14 +160,14 @@ func nodeFactoryFunc(flagPrefix, kind string) func(*launcher.Runtime) (launcher. blockStreamServer := blockstream.NewUnmanagedServer(blockstream.ServerOptionWithLogger(appLogger)) oneBlockStoreURL := mustReplaceDataDir(sfDataDir, viper.GetString("common-one-blocks-store-url")) mergedBlockStoreURL := mustReplaceDataDir(sfDataDir, viper.GetString("common-merged-blocks-store-url")) - workingDir := mustReplaceDataDir(sfDataDir, viper.GetString("mindreader-node-working-dir")) - gprcListenAdrr := viper.GetString("mindreader-node-grpc-listen-addr") - mergeThresholdBlockAge := viper.GetString("mindreader-node-merge-threshold-block-age") - waitTimeForUploadOnShutdown := viper.GetDuration("mindreader-node-wait-upload-complete-on-shutdown") - oneBlockFileSuffix := viper.GetString("mindreader-node-one-block-suffix") - blocksChanCapacity := viper.GetInt("mindreader-node-blocks-chan-capacity") - - mindreaderPlugin, err := getMindreaderLogPlugin( + workingDir := mustReplaceDataDir(sfDataDir, viper.GetString("reader-node-working-dir")) + gprcListenAdrr := viper.GetString("reader-node-grpc-listen-addr") + mergeThresholdBlockAge := viper.GetString("reader-node-merge-threshold-block-age") + waitTimeForUploadOnShutdown := viper.GetDuration("reader-node-wait-upload-complete-on-shutdown") + oneBlockFileSuffix := viper.GetString("reader-node-one-block-suffix") + blocksChanCapacity := viper.GetInt("reader-node-blocks-chan-capacity") + + readerPlugin, err := getReaderLogPlugin( blockStreamServer, oneBlockStoreURL, mergedBlockStoreURL, @@ -184,17 +184,17 @@ func nodeFactoryFunc(flagPrefix, kind string) func(*launcher.Runtime) (launcher. appTracer, ) if err != nil { - return nil, fmt.Errorf("new mindreader plugin: %w", err) + return nil, fmt.Errorf("new reader plugin: %w", err) } - superviser.RegisterLogPlugin(mindreaderPlugin) + superviser.RegisterLogPlugin(readerPlugin) return nodeManagerApp.New(&nodeManagerApp.Config{ HTTPAddr: httpAddr, GRPCAddr: gprcListenAdrr, }, &nodeManagerApp.Modules{ Operator: chainOperator, - MindreaderPlugin: mindreaderPlugin, + MindreaderPlugin: readerPlugin, MetricsAndReadinessManager: metricsAndReadinessManager, RegisterGRPCService: func(server *grpc.Server) error { pbheadinfo.RegisterHeadInfoServer(server, blockStreamServer) @@ -236,7 +236,7 @@ func buildNodeArguments(nodeDataDir, nodeRole string, endpoints []string, start, } typeRoles := nodeArgsByRole{ - "mindreader": strings.Join(thegariiArgs, " "), + "reader": strings.Join(thegariiArgs, " "), } argsString, ok := typeRoles[nodeRole] diff --git a/cmd/firearweave/cli/relayer.go b/cmd/firearweave/cli/relayer.go index e731343..88e279c 100644 --- a/cmd/firearweave/cli/relayer.go +++ b/cmd/firearweave/cli/relayer.go @@ -17,7 +17,7 @@ func init() { Description: "Serves blocks as a stream, with a buffer", RegisterFlags: func(cmd *cobra.Command) error { cmd.Flags().String("relayer-grpc-listen-addr", RelayerServingAddr, "Address to listen for incoming gRPC requests") - cmd.Flags().StringSlice("relayer-source", []string{MindreaderGRPCAddr}, "List of Blockstream sources (mindreaders) to connect to for live block feeds (repeat flag as needed)") + cmd.Flags().StringSlice("relayer-source", []string{ReaderGRPCAddr}, "List of Blockstream sources (readers) to connect to for live block feeds (repeat flag as needed)") cmd.Flags().Duration("relayer-max-source-latency", 999999*time.Hour, "Max latency tolerated to connect to a source. A performance optimization for when you have redundant sources and some may not have caught up") return nil }, diff --git a/codec/consolereader_test.go b/codec/consolereader_test.go new file mode 100644 index 0000000..d3ccb65 --- /dev/null +++ b/codec/consolereader_test.go @@ -0,0 +1,152 @@ +// Copyright 2021 dfuse Platform Inc. // +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codec + +import ( + "bytes" + "encoding/json" + "io" + "io/ioutil" + "os" + "os/exec" + "reflect" + "strings" + "testing" + + pbarweave "github.com/streamingfast/firehose-arweave/types/pb/sf/arweave/type/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +type ObjectReader func() (interface{}, error) + +func TestParseFromFile(t *testing.T) { + zlog, _ = zap.NewDevelopment() + tests := []struct { + firehoseFile string + expectedPanicErr error + }{ + // Skipping as the data is broken + // {"testdata/full.firelog", nil}, + } + + for _, test := range tests { + t.Run(strings.Replace(test.firehoseFile, "testdata/", "", 1), func(t *testing.T) { + defer func() { + if r := recover(); r != nil { + require.Equal(t, test.expectedPanicErr, r) + } + }() + + cr := testFileConsoleReader(t, test.firehoseFile) + buf := &bytes.Buffer{} + buf.Write([]byte("[")) + + for first := true; true; first = false { + out, err := cr.ReadBlock() + if err == io.EOF { + break + } + require.NoError(t, err) + + block := out.ToProtocol().(*pbarweave.Block) + + if !isNil(block) { + if !first { + buf.Write([]byte(",")) + } + + // FIXMME: jsonpb needs to be updated to latest version of used gRPC + // elements. We are disaligned and using that breaks now. + // Needs to check what is the latest way to properly serialize + // Proto generated struct to JSON. + // value, err := jsonpb.MarshalIndentToString(v, " ") + // require.NoError(t, err) + + value, err := json.MarshalIndent(block, "", " ") + require.NoError(t, err) + + buf.Write(value) + } + + if len(buf.Bytes()) != 0 { + buf.Write([]byte("\n")) + } + + } + buf.Write([]byte("]")) + + goldenFile := test.firehoseFile + ".golden.json" + if os.Getenv("GOLDEN_UPDATE") == "true" { + ioutil.WriteFile(goldenFile, buf.Bytes(), os.ModePerm) + } + + cnt, err := ioutil.ReadFile(goldenFile) + require.NoError(t, err) + + if !assert.Equal(t, string(cnt), buf.String()) { + t.Error("previous diff:\n" + unifiedDiff(t, cnt, buf.Bytes())) + } + }) + } +} + +func isNil(v interface{}) bool { + if v == nil { + return true + } + + rv := reflect.ValueOf(v) + return rv.Kind() == reflect.Ptr && rv.IsNil() +} + +func testFileConsoleReader(t *testing.T, filename string) *ConsoleReader { + t.Helper() + + fl, err := os.Open(filename) + require.NoError(t, err) + + cr := testReaderConsoleReader(t, make(chan string, 10000), func() { fl.Close() }) + + go cr.ProcessData(fl) + + return cr +} + +func testReaderConsoleReader(t *testing.T, lines chan string, closer func()) *ConsoleReader { + t.Helper() + + l := &ConsoleReader{ + lines: lines, + close: closer, + } + + return l +} + +func unifiedDiff(t *testing.T, cnt1, cnt2 []byte) string { + file1 := "/tmp/gotests-linediff-1" + file2 := "/tmp/gotests-linediff-2" + err := ioutil.WriteFile(file1, cnt1, 0600) + require.NoError(t, err) + + err = ioutil.WriteFile(file2, cnt2, 0600) + require.NoError(t, err) + + cmd := exec.Command("diff", "-u", file1, file2) + out, _ := cmd.Output() + + return string(out) +} diff --git a/devel/live-blocks/live-blocks.yaml b/devel/live-blocks/live-blocks.yaml index 570f48d..b6cc1a6 100644 --- a/devel/live-blocks/live-blocks.yaml +++ b/devel/live-blocks/live-blocks.yaml @@ -1,7 +1,7 @@ start: args: - merger - - mindreader-node + - reader-node flags: - mindreader-node-path: "thegarii" - mindreader-node-merge-threshold-block-age: never + reader-node-path: "thegarii" + reader-node-merge-threshold-block-age: never diff --git a/devel/reproc-mainnet/reproc-mainnet.yaml b/devel/reproc-mainnet/reproc-mainnet.yaml index c725874..ca03bb2 100644 --- a/devel/reproc-mainnet/reproc-mainnet.yaml +++ b/devel/reproc-mainnet/reproc-mainnet.yaml @@ -1,9 +1,9 @@ start: args: - - mindreader-node + - reader-node flags: # common-one-blocks-store-url: gs://dfuseio-global-blocks-us/arweave-mainnet/v1-oneblock - mindreader-node-path: "thegarii" - mindreader-node-merge-threshold-block-age: always - mindreader-node-start-block-num: 928200 - mindreader-node-stop-block-num: 928500 + reader-node-path: "thegarii" + reader-node-merge-threshold-block-age: always + reader-node-start-block-num: 928200 + reader-node-stop-block-num: 928500 diff --git a/devel/standard/standard.yaml b/devel/standard/standard.yaml index 2a8da77..a26b9cf 100644 --- a/devel/standard/standard.yaml +++ b/devel/standard/standard.yaml @@ -2,16 +2,16 @@ start: args: - relayer - merger - - mindreader-node + - reader-node - firehose flags: common-live-source-addr: "localhost:16011" # Specifies the path to the dummy chain binary is - mindreader-node-path: "thegarii" + reader-node-path: "thegarii" # debugging - mindreader-node-debug-deep-mind: false - mindreader-node-log-to-zap: true - mindreader-node-merge-threshold-block-age: never + reader-node-debug-firehose-logs: false + reader-node-log-to-zap: true + reader-node-merge-threshold-block-age: never # Once fully live with chain, those should be removed, they are used so that Firehose serves # blocks even if the chain is not live yet. relayer-max-source-latency: 999999999s diff --git a/nodemanager/logging.go b/nodemanager/logging.go index a32f82f..2109855 100644 --- a/nodemanager/logging.go +++ b/nodemanager/logging.go @@ -8,7 +8,7 @@ import ( "go.uber.org/zap/zapcore" ) -// This file configures a logging extractor that transforms log lines received from the blockchain process running +// This file configures a logging reader that transforms log lines received from the blockchain process running // and then logs them inside the Firehose stack logging system. // // A simple regex is going to identify the level of the line and turn it into our internal level value. @@ -21,11 +21,11 @@ import ( // So our regex look like the one below, extracting the `info` value from a group in the regexp. var logLevelRegex = regexp.MustCompile("level=(debug|info|warn|warning|error)") -func newToZapLogPlugin(debugDeepMind bool, logger *zap.Logger) *logplugin.ToZapLogPlugin { - return logplugin.NewToZapLogPlugin(debugDeepMind, logger, logplugin.ToZapLogPluginLogLevel(logLevelExtractor), logplugin.ToZapLogPluginTransformer(stripTimeTransformer)) +func newToZapLogPlugin(debugFirehoseLogs bool, logger *zap.Logger) *logplugin.ToZapLogPlugin { + return logplugin.NewToZapLogPlugin(debugFirehoseLogs, logger, logplugin.ToZapLogPluginLogLevel(logLevelReader), logplugin.ToZapLogPluginTransformer(stripTimeTransformer)) } -func logLevelExtractor(in string) zapcore.Level { +func logLevelReader(in string) zapcore.Level { // If the regex does not match the line, log to `INFO` so at least we see something by default. groups := logLevelRegex.FindStringSubmatch(in) if len(groups) <= 1 { diff --git a/nodemanager/superviser.go b/nodemanager/superviser.go index 98ede28..4f186f3 100644 --- a/nodemanager/superviser.go +++ b/nodemanager/superviser.go @@ -39,12 +39,12 @@ func NewSuperviser( arguments []string, dataDir string, headBlockUpdateFunc nodeManager.HeadBlockUpdater, - debugDeepMind bool, + debugFirehoseLogs bool, logToZap bool, appLogger *zap.Logger, nodelogger *zap.Logger, ) *Superviser { - // Ensure process manager line buffer is large enough (50 MiB) for our Deep Mind instrumentation outputting lot's of text. + // Ensure process manager line buffer is large enough (50 MiB) for our Firehose instrumentation outputting lot's of text. overseer.DEFAULT_LINE_BUFFER_SIZE = 50 * 1024 * 1024 supervisor := &Superviser{ @@ -59,9 +59,9 @@ func NewSuperviser( supervisor.RegisterLogPlugin(logplugin.LogPluginFunc(supervisor.lastBlockSeenLogPlugin)) if logToZap { - supervisor.RegisterLogPlugin(newToZapLogPlugin(debugDeepMind, nodelogger)) + supervisor.RegisterLogPlugin(newToZapLogPlugin(debugFirehoseLogs, nodelogger)) } else { - supervisor.RegisterLogPlugin(logplugin.NewToConsoleLogPlugin(debugDeepMind)) + supervisor.RegisterLogPlugin(logplugin.NewToConsoleLogPlugin(debugFirehoseLogs)) } appLogger.Info("created arweave superviser", zap.Object("superviser", supervisor)) @@ -115,12 +115,12 @@ func (s *Superviser) MarshalLogObject(enc zapcore.ObjectEncoder) error { } func (s *Superviser) lastBlockSeenLogPlugin(line string) { - // DMLOG BLOCK ... - if !strings.HasPrefix(line, "DMLOG BLOCK") { + // FIRE BLOCK ... + if !strings.HasPrefix(line, "FIRE BLOCK") { return } - blockNumStr := line[12:] + blockNumStr := line[11:] nextSpace := strings.Index(blockNumStr, " ") if nextSpace < 0 { s.Logger.Error("unable to extract last block num, missing space", zap.String("line", line)) @@ -142,7 +142,7 @@ func (s *Superviser) lastBlockSeenLogPlugin(line string) { // FIXME: Instrumentation needs to always have a way to easily decode height, // hash and timestamp. Right now in Arweave, we have only the height. // - // It's not important for now because only mindreaders are running and those + // It's not important for now because only readers are running and those // updates are carried on directly by the console reader. // s.headBlockUpdateFunc(s.lastBlockSeen, diff --git a/tools/fireacme/99-firehose-acme.sh b/tools/fireacme/99-firehose-acme.sh index 7c0d66e..af8ab56 100644 --- a/tools/fireacme/99-firehose-acme.sh +++ b/tools/fireacme/99-firehose-acme.sh @@ -6,7 +6,7 @@ export PATH=$PATH:/app # If we are in a "node-manager" image, display special scripts motd# # -# *Note* Our (i.e. firehose-acme) Mindreader data directory is at the root +# *Note* Our (i.e. firehose-acme) Reader data directory is at the root # `/data` mount point. Inside it, the `dummmy-blockchain` binary # itself creates a `data` subfolder. This is why we have `/data/data` # here as the path to check if we are inside a Node Manager instance diff --git a/tools/fireacme/motd_node_manager b/tools/fireacme/motd_node_manager index 8c9844f..6028327 100644 --- a/tools/fireacme/motd_node_manager +++ b/tools/fireacme/motd_node_manager @@ -9,7 +9,7 @@ Welcome to this node manager pod, common commands: Node manager commands (acme-... to show all): - Check if running acme-is-running - - Debug Deep Mind logs for 30s acme-debug-deep-mind-30s + - Debug Firehose logs for 30s acme-debug-firehose-logs-30s - Stop instance acme-maintenance - Start or restart instance acme-resume diff --git a/tools/fireacme/scripts/acme-debug-deep-mind-30s b/tools/fireacme/scripts/acme-debug-firehose-logs-30s similarity index 58% rename from tools/fireacme/scripts/acme-debug-deep-mind-30s rename to tools/fireacme/scripts/acme-debug-firehose-logs-30s index 7d441b6..14c2adf 100755 --- a/tools/fireacme/scripts/acme-debug-deep-mind-30s +++ b/tools/fireacme/scripts/acme-debug-firehose-logs-30s @@ -2,12 +2,12 @@ port=${MANAGER_API_PORT:-8080} -# description: Stop geth for maintenance, start it back with debug deep mind enabled, wait 60s and remove the flag +# description: Stop geth for maintenance, start it back with debug Firehose enabled, wait 60s and remove the flag curl -sS -XPOST localhost:$port/v1/maintenance?sync=true -curl -sS -XPOST -d "debug-deep-mind=true" localhost:$port/v1/resume?sync=true +curl -sS -XPOST -d "debug-firehose-logs=true" localhost:$port/v1/resume?sync=true # We give 10s more than 30s to account for startup time of geth sleep 40s curl -sS -XPOST localhost:$port/v1/maintenance?sync=true -curl -sS -XPOST -d "debug-deep-mind=false" localhost:$port/v1/resume?sync=true +curl -sS -XPOST -d "debug-firehose-logs=false" localhost:$port/v1/resume?sync=true From d07d26631b7b62b08264f881ed2f28b2f147c070 Mon Sep 17 00:00:00 2001 From: emiliocramer Date: Thu, 15 Sep 2022 12:31:13 -0400 Subject: [PATCH 04/11] common-one blocks -> block --- cmd/firearweave/cli/common-flag.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/firearweave/cli/common-flag.go b/cmd/firearweave/cli/common-flag.go index deb1afb..4553485 100644 --- a/cmd/firearweave/cli/common-flag.go +++ b/cmd/firearweave/cli/common-flag.go @@ -23,9 +23,9 @@ import ( func init() { launcher.RegisterCommonFlags = func(logger *zap.Logger, cmd *cobra.Command) error { //Common stores configuration flags - cmd.Flags().String("common-merged-blocks-store-url", MergedBlocksStoreURL, "[COMMON] Store URL (with prefix) where to read/write. Used by: mindreader, merger, relayer and firehose") + cmd.Flags().String("common-merged-blocks-store-url", MergedBlocksStoreURL, "[COMMON] Store URL (with prefix) where to read/write. Used by: reader, merger, relayer and firehose") cmd.Flags().String("common-forked-blocks-store-url", ForkedBlockStoreURL, "[COMMON] Store URL (with prefix) to read/write forked block files. Used by: merger, firehose") - cmd.Flags().String("common-one-blocks-store-url", OneBlockStoreURL, "[COMMON] Store URL (with prefix) to read/write one-block files. Used by: mindreader, merger") + cmd.Flags().String("common-one-block-store-url", OneBlockStoreURL, "[COMMON] Store URL (with prefix) to read/write one-block files. Used by: reader, merger") cmd.Flags().String("common-live-source-addr", RelayerServingAddr, "[COMMON] gRPC endpoint to get real-time blocks. Used by: firehose") cmd.Flags().Bool("common-blocks-cache-enabled", false, "[COMMON] enable ATM caching") From f6cb109c037e027c5aa1b8bf3a65b92ae4c7d423 Mon Sep 17 00:00:00 2001 From: emiliocramer Date: Thu, 15 Sep 2022 12:31:27 -0400 Subject: [PATCH 05/11] DMLOG -> FIRE --- nodemanager/superviser_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nodemanager/superviser_test.go b/nodemanager/superviser_test.go index f0d4490..2c48584 100644 --- a/nodemanager/superviser_test.go +++ b/nodemanager/superviser_test.go @@ -13,12 +13,12 @@ func TestSuperviser_lastBlockSeenLogPlugin(t *testing.T) { line string want uint64 }{ - {"block line zero", "DMLOG BLOCK 0 5b02121274e67b59671b7e6c3711cc74", 0}, - {"block line some", "DMLOG BLOCK 10 5b02121274e67b59671b7e6c3711cc74", 10}, - {"block line max", "DMLOG BLOCK 18446744073709551615 5b02121274e67b59671b7e6c3711cc74", uint64(math.MaxUint64)}, + {"block line zero", "FIRE BLOCK 0 5b02121274e67b59671b7e6c3711cc74", 0}, + {"block line some", "FIRE BLOCK 10 5b02121274e67b59671b7e6c3711cc74", 10}, + {"block line max", "FIRE BLOCK 18446744073709551615 5b02121274e67b59671b7e6c3711cc74", uint64(math.MaxUint64)}, // Only logs an error - {"block line missing data", "DMLOG BLOCK 10", 0}, + {"block line missing data", "FIRE BLOCK 10", 0}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 52367c54a361143d76526572742784efe218b377 Mon Sep 17 00:00:00 2001 From: emiliocramer Date: Thu, 15 Sep 2022 12:32:19 -0400 Subject: [PATCH 06/11] sfarweave binn -> firearweave --- devel/live-blocks/start.sh | 4 ++-- devel/reproc-mainnet/start.sh | 4 ++-- devel/standard/start.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/devel/live-blocks/start.sh b/devel/live-blocks/start.sh index a8b6d77..6e293fc 100755 --- a/devel/live-blocks/start.sh +++ b/devel/live-blocks/start.sh @@ -3,7 +3,7 @@ ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" clean= -sfarweave="$ROOT/../firearweave" +firearweave="$ROOT/../firearweave" main() { pushd "$ROOT" &> /dev/null @@ -27,7 +27,7 @@ main() { # check if thegarii exists check_thegarii - exec $sfarweave -c $(basename $ROOT).yaml start "$@" + exec $firearweave -c $(basename $ROOT).yaml start "$@" } usage_error() { diff --git a/devel/reproc-mainnet/start.sh b/devel/reproc-mainnet/start.sh index a8b6d77..6e293fc 100755 --- a/devel/reproc-mainnet/start.sh +++ b/devel/reproc-mainnet/start.sh @@ -3,7 +3,7 @@ ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" clean= -sfarweave="$ROOT/../firearweave" +firearweave="$ROOT/../firearweave" main() { pushd "$ROOT" &> /dev/null @@ -27,7 +27,7 @@ main() { # check if thegarii exists check_thegarii - exec $sfarweave -c $(basename $ROOT).yaml start "$@" + exec $firearweave -c $(basename $ROOT).yaml start "$@" } usage_error() { diff --git a/devel/standard/start.sh b/devel/standard/start.sh index a8b6d77..6e293fc 100755 --- a/devel/standard/start.sh +++ b/devel/standard/start.sh @@ -3,7 +3,7 @@ ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" clean= -sfarweave="$ROOT/../firearweave" +firearweave="$ROOT/../firearweave" main() { pushd "$ROOT" &> /dev/null @@ -27,7 +27,7 @@ main() { # check if thegarii exists check_thegarii - exec $sfarweave -c $(basename $ROOT).yaml start "$@" + exec $firearweave -c $(basename $ROOT).yaml start "$@" } usage_error() { From 3ef5d472ea9477d9e28e204a105a8a811b16b8b3 Mon Sep 17 00:00:00 2001 From: emiliocramer Date: Thu, 15 Sep 2022 12:41:36 -0400 Subject: [PATCH 07/11] add changelog --- CHANGELOG.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e69de29 From c68190f7ae5378c19c00e0797c541172b63216d1 Mon Sep 17 00:00:00 2001 From: emiliocramer Date: Thu, 15 Sep 2022 12:43:16 -0400 Subject: [PATCH 08/11] fill changelog --- CHANGELOG.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e69de29..b412287 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -0,0 +1,29 @@ +# Change log + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this +project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). See [MAINTAINERS.md](./MAINTAINERS.md) +for instructions to keep up to date. + + +## v1.0.0 + + +#### Project Rename + +* The binary name has changed from `sfarweave` to `firearweave` (aligned with https://firehose.streamingfast.io/references/naming-conventions) + +#### Flags and environment variables + +* Renamed the `mindreader` application to `reader` +* Renamed `common-blocks-store-url` to `common-block-store-url` +* Renamed `deepmind` to `firehose` +* Renamed `debug-deep-mind` flags to `debug-firehose-logs` +* Renamed `extractor` to `reader` +* Renamed `DMLOG` prefix to `FIRE` +* Renamed `dmlog` to `firelog` +* Renamed all the `mindreader-node-*` flags to `reader-node-*` + + + + + From 79811a788ff1682816bed8b68af5068dca752bfe Mon Sep 17 00:00:00 2001 From: emiliocramer Date: Wed, 21 Sep 2022 13:56:25 -0400 Subject: [PATCH 09/11] small changes/typos --- cmd/firearweave/cli/main.go | 2 +- cmd/firearweave/cli/node.go | 2 +- tools/fireacme/{99-firehose-acme.sh => 99-firehose-arweave.sh} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename tools/fireacme/{99-firehose-acme.sh => 99-firehose-arweave.sh} (100%) diff --git a/cmd/firearweave/cli/main.go b/cmd/firearweave/cli/main.go index 19e1ad8..7db9122 100644 --- a/cmd/firearweave/cli/main.go +++ b/cmd/firearweave/cli/main.go @@ -66,7 +66,7 @@ func Main() { derr.Check("arweave-blockchain", RootCmd.Execute()) } -var startCmdExample = `firearweave start reader` +var startCmdExample = `firearweave start reader-node` var startCmdHelpTemplate = `Usage:{{if .Runnable}} {{.UseLine}}{{end}} [all|command1 [command2...]]{{if gt (len .Aliases) 0}} diff --git a/cmd/firearweave/cli/node.go b/cmd/firearweave/cli/node.go index f6acdbb..af2224a 100644 --- a/cmd/firearweave/cli/node.go +++ b/cmd/firearweave/cli/node.go @@ -31,7 +31,7 @@ var readerArweaveLogger, readerArweaveTracer = logging.PackageLogger("reader.arw func registerCommonNodeFlags(cmd *cobra.Command, flagPrefix string, managerAPIAddr string) { cmd.Flags().String(flagPrefix+"path", "thegarii", FlagDescription(` - Process that will be invoked reader (a.k.a reader) component, can be a full path or just the binary's name, in which case the binary is + Process that will be invoked reader component, can be a full path or just the binary's name, in which case the binary is searched for paths listed by the PATH environment variable (following operating system rules around PATH handling). `)) cmd.Flags().String(flagPrefix+"data-dir", "{data-dir}/{node-role}/data", "Directory for node data ({node-role} is either reader, peering or dev-miner)") diff --git a/tools/fireacme/99-firehose-acme.sh b/tools/fireacme/99-firehose-arweave.sh similarity index 100% rename from tools/fireacme/99-firehose-acme.sh rename to tools/fireacme/99-firehose-arweave.sh From 39259c64c0c7089dcf9fb19f528d87bf9dc94482 Mon Sep 17 00:00:00 2001 From: emiliocramer Date: Wed, 21 Sep 2022 13:56:51 -0400 Subject: [PATCH 10/11] fireacme -> firearweave --- .gitignore | 8 ++++---- cmd/firearweave/cli/node.go | 2 +- cmd/firearweave/cli/reader-node.go | 2 +- tools/fireacme/99-firehose-arweave.sh | 2 +- tools/fireacme/motd_generic | 2 +- tools/fireacme/motd_node_manager | 14 +++++++------- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index cddecda..9805de4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ ms1.test.md .DS_Store # To facilitate development, you can use [direnv](https://direnv.net/) to auto-load -# some environment variables. We have in `devel` a shim script called directly `fireacme` +# some environment variables. We have in `devel` a shim script called directly `firearweave` # (that should have been renamed to your specific chain like `fireeth`). This small shim # compiles the project and then invoke resulting binary. This means that when using this # .envrc file: @@ -14,14 +14,14 @@ ms1.test.md # export PATH="`pwd`/devel:$PATH" # ``` # -# Each time you enter the project in your terminal (e.g. `cd ~/work/firehose-acme`) and then +# Each time you enter the project in your terminal (e.g. `cd ~/work/firehose-arweave`) and then # do `firehose start --help`, then it first compiles all the Go source code and then invoke the # resulting just compiled binary, meaning you are "always" freshly compiled, it's more or less -# equivalent to doing directly `go run ./cmd/fireacme start --help` (which is totally fine). +# equivalent to doing directly `go run ./cmd/firearweave start --help` (which is totally fine). .envrc* sf-data -/fireacme +/firearweave /dist fire-data \ No newline at end of file diff --git a/cmd/firearweave/cli/node.go b/cmd/firearweave/cli/node.go index af2224a..76df7ff 100644 --- a/cmd/firearweave/cli/node.go +++ b/cmd/firearweave/cli/node.go @@ -41,7 +41,7 @@ func registerCommonNodeFlags(cmd *cobra.Command, flagPrefix string, managerAPIAd is intercepted, split line by line and each line is then transformed and logged through the Firehose stack logging system. The transformation extracts the level and remove the timestamps creating a 'sanitized' version of the logs emitted by the blockchain's managed client process. If this is not desirable, disabled the flag - and all the invoked process standard error will be redirect to 'fireacme' standard's output. + and all the invoked process standard error will be redirect to 'firearweave' standard's output. `, flagPrefix+"path")) cmd.Flags().String(flagPrefix+"manager-api-addr", managerAPIAddr, "Arweave node manager API address") cmd.Flags().Duration(flagPrefix+"readiness-max-latency", 10*time.Minute, "Determine the maximum head block latency at which the instance will be determined healthy. Some chains have more regular block production than others.") diff --git a/cmd/firearweave/cli/reader-node.go b/cmd/firearweave/cli/reader-node.go index 48eddf7..c402f78 100644 --- a/cmd/firearweave/cli/reader-node.go +++ b/cmd/firearweave/cli/reader-node.go @@ -40,7 +40,7 @@ func registerReaderNodeFlags(cmd *cobra.Command) error { cmd.Flags().Uint("reader-node-stop-block-num", 0, "Shutdown reader when we the following 'stop-block-num' has been reached, inclusively.") cmd.Flags().Int("reader-node-blocks-chan-capacity", 100, "Capacity of the channel holding blocks read by the reader. Process will shutdown superviser/geth if the channel gets over 90% of that capacity to prevent horrible consequences. Raise this number when processing tiny blocks very quickly") cmd.Flags().String("reader-node-one-block-suffix", "default", FlagDescription(` - Unique identifier for Reader, so that it can produce 'oneblock files' in the same store as another instance without competing + Unique identifier for Reader, so that it can produce 'one block files' in the same store as another instance without competing for writes. You should set this flag if you have multiple reader running, each one should get a unique identifier, the hostname value is a good value to use. `)) diff --git a/tools/fireacme/99-firehose-arweave.sh b/tools/fireacme/99-firehose-arweave.sh index af8ab56..027ffde 100644 --- a/tools/fireacme/99-firehose-arweave.sh +++ b/tools/fireacme/99-firehose-arweave.sh @@ -1,5 +1,5 @@ ## -# This is place inside `/etc/profile.d/99-firehose-acme.sh` +# This is place inside `/etc/profile.d/99-firehose-arweave.sh` # on built system an executed to provide message to use when they # connect on the box. export PATH=$PATH:/app diff --git a/tools/fireacme/motd_generic b/tools/fireacme/motd_generic index 79dfb89..fed6f71 100644 --- a/tools/fireacme/motd_generic +++ b/tools/fireacme/motd_generic @@ -4,5 +4,5 @@ /_/ /_/_/ \__/_//_/\___/___/\__/ Welcome to this pod, common commands: - - Show version fireacme --version + - Show version firearweave --version diff --git a/tools/fireacme/motd_node_manager b/tools/fireacme/motd_node_manager index 6028327..65f0f1c 100644 --- a/tools/fireacme/motd_node_manager +++ b/tools/fireacme/motd_node_manager @@ -4,13 +4,13 @@ /_/ /_/_/ \__/_//_/\___/___/\__/ Welcome to this node manager pod, common commands: - - Show version fireacme --version + - Show version firearweave --version -Node manager commands (acme-... to show all): - - Check if running acme-is-running +Node manager commands (arweave-... to show all): + - Check if running arweave-is-running - - Debug Firehose logs for 30s acme-debug-firehose-logs-30s - - Stop instance acme-maintenance - - Start or restart instance acme-resume + - Debug Firehose logs for 30s arweave-debug-firehose-logs-30s + - Stop instance arweave-maintenance + - Start or restart instance arweave-resume - - RPC Head Block acme-rpc-head-block + - RPC Head Block arweave-rpc-head-block From 806ea588eb37118a03efe022b639ca5b9c38a74d Mon Sep 17 00:00:00 2001 From: emiliocramer Date: Wed, 21 Sep 2022 13:57:25 -0400 Subject: [PATCH 11/11] reduce changelog --- CHANGELOG.md | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b412287..a54c0be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,20 +7,10 @@ for instructions to keep up to date. ## v1.0.0 - -#### Project Rename - -* The binary name has changed from `sfarweave` to `firearweave` (aligned with https://firehose.streamingfast.io/references/naming-conventions) - #### Flags and environment variables * Renamed the `mindreader` application to `reader` -* Renamed `common-blocks-store-url` to `common-block-store-url` -* Renamed `deepmind` to `firehose` -* Renamed `debug-deep-mind` flags to `debug-firehose-logs` -* Renamed `extractor` to `reader` -* Renamed `DMLOG` prefix to `FIRE` -* Renamed `dmlog` to `firelog` +* Renamed `common-one-blocks-store-url` to `common-one-block-store-url` * Renamed all the `mindreader-node-*` flags to `reader-node-*`