fixes

gobitfly · Nov 6, 2024 · cd8bbcb · cd8bbcb
1 parent 2bcd424
commit cd8bbcb
Show file tree

Hide file tree

Showing 8 changed files with 29 additions and 20 deletions.
diff --git a/backend/pkg/commons/db2/client_test.go b/backend/pkg/commons/db2/client_test.go
@@ -166,7 +166,7 @@ func TestBigTableClient(t *testing.T) {
 	}
 
 	client, admin := storetest.NewBigTable(t)
-	bg, err := store.NewBigTableWithClient(context.Background(), client, admin, raw)
+	bg, err := store.NewBigTableWithClient(context.Background(), client, admin, RawSchema)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -230,7 +230,7 @@ func TestBigTableClientWithFallback(t *testing.T) {
 	}
 
 	client, admin := storetest.NewBigTable(t)
-	bg, err := store.NewBigTableWithClient(context.Background(), client, admin, raw)
+	bg, err := store.NewBigTableWithClient(context.Background(), client, admin, RawSchema)
 	if err != nil {
 		t.Fatal(err)
 	}

diff --git a/backend/pkg/commons/db2/parser.go b/backend/pkg/commons/db2/parser.go
@@ -96,7 +96,7 @@ type txExtraInfo struct {
 
 // parseEthBlock is a copy of ethclient.Client.getBlock
 // modified to work the with raw db
-// https://github.com/ethereum/go-ethereum/blob/v1.14.11/ethclient/ethclient.go#L129
+// https://github.com/ethereum/go-ethereum/blob/v1.13.12/ethclient/ethclient.go#L129
 func parseEthBlock(raw json.RawMessage, rawUncles []jsonrpcMessage) (*types.Block, error) {
 	// Decode header and transactions.
 	var head *types.Header

diff --git a/backend/pkg/commons/db2/raw.go b/backend/pkg/commons/db2/raw.go
@@ -137,9 +137,8 @@ func (db RawStore) ReadBlocksByNumber(chainID uint64, start, end int64) ([]*Full
 		return nil, err
 	}
 	blocks := make([]*FullBlockRawData, 0, end-start+1)
-	for key, data := range rows {
-		number := blockKeyToNumber(chainID, key)
-		block, err := db.parseRow(chainID, number, data)
+	for _, row := range rows {
+		block, err := db.parseRow(chainID, blockKeyToNumber(chainID, row.Key), row.Values)
 		if err != nil {
 			return nil, err
 		}

diff --git a/backend/pkg/commons/db2/raw_test.go b/backend/pkg/commons/db2/raw_test.go
@@ -15,7 +15,7 @@ import (
 func TestRaw(t *testing.T) {
 	client, admin := storetest.NewBigTable(t)
 
-	s, err := store.NewBigTableWithClient(context.Background(), client, admin, raw)
+	s, err := store.NewBigTableWithClient(context.Background(), client, admin, RawSchema)
 	if err != nil {
 		t.Fatal(err)
 	}

diff --git a/backend/pkg/commons/db2/store/bigtable.go b/backend/pkg/commons/db2/store/bigtable.go
@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"cloud.google.com/go/bigtable"
-	"golang.org/x/exp/maps"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )
@@ -56,7 +55,7 @@ func (w TableWrapper) BulkAdd(itemsByKey map[string][]Item) error {
 	return w.BigTableStore.BulkAdd(w.table, itemsByKey)
 }
 
-func (w TableWrapper) GetRowsRange(high, low string) (map[string]map[string][]byte, error) {
+func (w TableWrapper) GetRowsRange(high, low string) ([]Row, error) {
 	return w.BigTableStore.GetRowsRange(w.table, high, low)
 }
 
@@ -148,15 +147,17 @@ func (b BigTableStore) BulkAdd(table string, itemsByKey map[string][]Item) error
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
+	var keys []string
 	var muts []*bigtable.Mutation
-	for _, items := range itemsByKey {
+	for key, items := range itemsByKey {
 		mut := bigtable.NewMutation()
 		for _, item := range items {
 			mut.Set(item.Family, item.Column, bigtable.Timestamp(0), item.Data)
 		}
+		keys = append(keys, key)
 		muts = append(muts, mut)
 	}
-	errs, err := tbl.ApplyBulk(ctx, maps.Keys(itemsByKey), muts)
+	errs, err := tbl.ApplyBulk(ctx, keys, muts)
 	if err != nil {
 		return fmt.Errorf("cannot ApplyBulk err: %w", err)
 	}
@@ -259,20 +260,29 @@ func (b BigTableStore) GetRow(table, key string) (map[string][]byte, error) {
 	return data, nil
 }
 
-func (b BigTableStore) GetRowsRange(table, high, low string) (map[string]map[string][]byte, error) {
+type Row struct {
+	Key    string
+	Values map[string][]byte
+}
+
+func (b BigTableStore) GetRowsRange(table, high, low string) ([]Row, error) {
 	tbl := b.client.Open(table)
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
 	rowRange := bigtable.NewClosedRange(low, high)
-	data := make(map[string]map[string][]byte)
+	var data []Row
 	err := tbl.ReadRows(ctx, rowRange, func(row bigtable.Row) bool {
-		data[row.Key()] = make(map[string][]byte)
+		values := make(map[string][]byte)
 		for _, family := range row {
 			for _, item := range family {
-				data[row.Key()][item.Column] = item.Value
+				values[item.Column] = item.Value
 			}
 		}
+		data = append(data, Row{
+			Key:    row.Key(),
+			Values: values,
+		})
 		return true
 	})
 

diff --git a/backend/pkg/commons/db2/store/remote.go b/backend/pkg/commons/db2/store/remote.go
@@ -142,7 +142,7 @@ func (r RemoteClient) GetLatestValue(key string) ([]byte, error) {
 	panic("implement me")
 }
 
-func (r RemoteClient) GetRowsRange(high, low string) (map[string]map[string][]byte, error) {
+func (r RemoteClient) GetRowsRange(high, low string) ([]Row, error) {
 	b, err := json.Marshal(ParamsGetRowsRange{
 		High: high,
 		Low:  low,
@@ -163,7 +163,7 @@ func (r RemoteClient) GetRowsRange(high, low string) (map[string]map[string][]by
 		b, _ := io.ReadAll(resp.Body)
 		return nil, fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, b)
 	}
-	var rows map[string]map[string][]byte
+	var rows []Row
 	if err := json.NewDecoder(resp.Body).Decode(&rows); err != nil {
 		return nil, err
 	}

diff --git a/backend/pkg/commons/db2/store/store.go b/backend/pkg/commons/db2/store/store.go
@@ -7,7 +7,7 @@ type Store interface {
 	GetRow(key string) (map[string][]byte, error)
 	GetRowKeys(prefix string) ([]string, error)
 	GetLatestValue(key string) ([]byte, error)
-	GetRowsRange(high, low string) (map[string]map[string][]byte, error)
+	GetRowsRange(high, low string) ([]Row, error)
 	Close() error
 	Clear() error
 }

diff --git a/backend/pkg/commons/db2/tables.go b/backend/pkg/commons/db2/tables.go
@@ -1,6 +1,6 @@
 package db2
 
-const BlocksRawTable = "blocks-raw"
+const BlocksRawTable = "blocks-RawSchema"
 
 const BT_COLUMNFAMILY_BLOCK = "b"
 const BT_COLUMN_BLOCK = "b"
@@ -13,7 +13,7 @@ const BT_COLUMN_UNCLES = "u"
 
 const MAX_EL_BLOCK_NUMBER = int64(1_000_000_000_000 - 1)
 
-var raw = map[string][]string{
+var RawSchema = map[string][]string{
 	BlocksRawTable: {
 		BT_COLUMNFAMILY_BLOCK,
 		BT_COLUMNFAMILY_RECEIPTS,