This repository has been archived by the owner on Aug 31, 2021. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 34
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #86 from vulcanize/vdb-371-recheck-queued-storage
(VDB-371) recheck queued storage
- Loading branch information
Showing
27 changed files
with
733 additions
and
236 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| // VulcanizeDB | ||
| // Copyright © 2019 Vulcanize | ||
|
|
||
| // This program is free software: you can redistribute it and/or modify | ||
| // it under the terms of the GNU Affero General Public License as published by | ||
| // the Free Software Foundation, either version 3 of the License, or | ||
| // (at your option) any later version. | ||
|
|
||
| // This program is distributed in the hope that it will be useful, | ||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| // GNU Affero General Public License for more details. | ||
|
|
||
| // You should have received a copy of the GNU Affero General Public License | ||
| // along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
|
|
||
| package fetcher | ||
|
|
||
| import ( | ||
| "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" | ||
| "github.com/vulcanize/vulcanizedb/pkg/fs" | ||
| "strings" | ||
| ) | ||
|
|
||
| type IStorageFetcher interface { | ||
| FetchStorageDiffs(chan<- utils.StorageDiffRow, chan<- error) | ||
| } | ||
|
|
||
| type CsvTailStorageFetcher struct { | ||
| tailer fs.Tailer | ||
| } | ||
|
|
||
| func NewCsvTailStorageFetcher(tailer fs.Tailer) CsvTailStorageFetcher { | ||
| return CsvTailStorageFetcher{tailer: tailer} | ||
| } | ||
|
|
||
| func (storageFetcher CsvTailStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffRow, errs chan<- error) { | ||
| t, tailErr := storageFetcher.tailer.Tail() | ||
| if tailErr != nil { | ||
| errs <- tailErr | ||
| } | ||
| for line := range t.Lines { | ||
| row, parseErr := utils.FromStrings(strings.Split(line.Text, ",")) | ||
| if parseErr != nil { | ||
| errs <- parseErr | ||
| } else { | ||
| out <- row | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| // VulcanizeDB | ||
| // Copyright © 2019 Vulcanize | ||
|
|
||
| // This program is free software: you can redistribute it and/or modify | ||
| // it under the terms of the GNU Affero General Public License as published by | ||
| // the Free Software Foundation, either version 3 of the License, or | ||
| // (at your option) any later version. | ||
|
|
||
| // This program is distributed in the hope that it will be useful, | ||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| // GNU Affero General Public License for more details. | ||
|
|
||
| // You should have received a copy of the GNU Affero General Public License | ||
| // along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
|
|
||
| package fetcher_test | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "strings" | ||
| "time" | ||
|
|
||
| "github.com/ethereum/go-ethereum/common" | ||
| "github.com/hpcloud/tail" | ||
| . "github.com/onsi/ginkgo" | ||
| . "github.com/onsi/gomega" | ||
|
|
||
| "github.com/vulcanize/vulcanizedb/libraries/shared/fetcher" | ||
| "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" | ||
| "github.com/vulcanize/vulcanizedb/pkg/fakes" | ||
| ) | ||
|
|
||
| var _ = Describe("Csv Tail Storage Fetcher", func() { | ||
| var ( | ||
| errorsChannel chan error | ||
| mockTailer *fakes.MockTailer | ||
| rowsChannel chan utils.StorageDiffRow | ||
| storageFetcher fetcher.CsvTailStorageFetcher | ||
| ) | ||
|
|
||
| BeforeEach(func() { | ||
| errorsChannel = make(chan error) | ||
| rowsChannel = make(chan utils.StorageDiffRow) | ||
| mockTailer = fakes.NewMockTailer() | ||
| storageFetcher = fetcher.NewCsvTailStorageFetcher(mockTailer) | ||
| }) | ||
|
|
||
| It("adds error to errors channel if tailing file fails", func(done Done) { | ||
| mockTailer.TailErr = fakes.FakeError | ||
|
|
||
| go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel) | ||
|
|
||
| Expect(<-errorsChannel).To(MatchError(fakes.FakeError)) | ||
| close(done) | ||
| }) | ||
|
|
||
| It("adds parsed csv row to rows channel for storage diff", func(done Done) { | ||
| line := getFakeLine() | ||
|
|
||
| go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel) | ||
| mockTailer.Lines <- line | ||
|
|
||
| expectedRow, err := utils.FromStrings(strings.Split(line.Text, ",")) | ||
| Expect(err).NotTo(HaveOccurred()) | ||
| Expect(<-rowsChannel).To(Equal(expectedRow)) | ||
| close(done) | ||
| }) | ||
|
|
||
| It("adds error to errors channel if parsing csv fails", func(done Done) { | ||
| line := &tail.Line{Text: "invalid"} | ||
|
|
||
| go storageFetcher.FetchStorageDiffs(rowsChannel, errorsChannel) | ||
| mockTailer.Lines <- line | ||
|
|
||
| Expect(<-errorsChannel).To(HaveOccurred()) | ||
| select { | ||
| case <-rowsChannel: | ||
| Fail("value passed to rows channel on error") | ||
| default: | ||
| Succeed() | ||
| } | ||
| close(done) | ||
| }) | ||
| }) | ||
|
|
||
| func getFakeLine() *tail.Line { | ||
| address := common.HexToAddress("0x1234567890abcdef") | ||
| blockHash := []byte{4, 5, 6} | ||
| blockHeight := int64(789) | ||
| storageKey := []byte{9, 8, 7} | ||
| storageValue := []byte{6, 5, 4} | ||
| return &tail.Line{ | ||
| Text: fmt.Sprintf("%s,%s,%d,%s,%s", common.Bytes2Hex(address.Bytes()), common.Bytes2Hex(blockHash), | ||
| blockHeight, common.Bytes2Hex(storageKey), common.Bytes2Hex(storageValue)), | ||
| Time: time.Time{}, | ||
| Err: nil, | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| // VulcanizeDB | ||
| // Copyright © 2019 Vulcanize | ||
|
|
||
| // This program is free software: you can redistribute it and/or modify | ||
| // it under the terms of the GNU Affero General Public License as published by | ||
| // the Free Software Foundation, either version 3 of the License, or | ||
| // (at your option) any later version. | ||
|
|
||
| // This program is distributed in the hope that it will be useful, | ||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| // GNU Affero General Public License for more details. | ||
|
|
||
| // You should have received a copy of the GNU Affero General Public License | ||
| // along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
|
|
||
| package mocks | ||
|
|
||
| import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" | ||
|
|
||
| type MockStorageFetcher struct { | ||
| RowsToReturn []utils.StorageDiffRow | ||
| ErrsToReturn []error | ||
| } | ||
|
|
||
| func NewMockStorageFetcher() *MockStorageFetcher { | ||
| return &MockStorageFetcher{} | ||
| } | ||
|
|
||
| func (fetcher *MockStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiffRow, errs chan<- error) { | ||
| defer close(out) | ||
| defer close(errs) | ||
| for _, err := range fetcher.ErrsToReturn { | ||
| errs <- err | ||
| } | ||
| for _, row := range fetcher.RowsToReturn { | ||
| out <- row | ||
| } | ||
| } |
Oops, something went wrong.