Merge remote-tracking branch 'origin/feature/snapshot' into develop

.gitignore

@@ -1,2 +1,3 @@
 .envrc
 .idea
+.DS_Store

snapshot/reader.go

@@ -92,7 +92,7 @@ func (r *Reader) Next() (*Section, error) {
 	r.nextOffset = uint64(beginOffset) + sectionSize + 8 // well well, sectionSize includes the rowCount I guess?
 
 	return &Section{
-		Name:       strings.TrimRight(str, string([]byte{0x00})),
+		Name:       stringToSectionName(strings.TrimRight(str, string([]byte{0x00}))),
 		Offset:     uint64(beginOffset),
 		Size:       sectionSize,
 		RowCount:   rowCount,

snapshot/section.go

@@ -2,8 +2,80 @@ package snapshot
 
 import "io"
 
+type SectionName string
+
+const (
+	SectionNameChainSnapshotHeader         SectionName = "eosio::chain::chain_snapshot_header"
+	SectionNameBlockState                  SectionName = "eosio::chain::block_state"
+	SectionNameAccountObject               SectionName = "eosio::chain::account_object"
+	SectionNameAccountMetadataObject       SectionName = "eosio::chain::account_metadata_object"
+	SectionNameAccountRamCorrectionObject  SectionName = "eosio::chain::account_ram_correction_object"
+	SectionNameGlobalPropertyObject        SectionName = "eosio::chain::global_property_object"
+	SectionNameProtocolStateObject         SectionName = "eosio::chain::protocol_state_object"
+	SectionNameDynamicGlobalPropertyObject SectionName = "eosio::chain::dynamic_global_property_object"
+	SectionNameBlockSummaryObject          SectionName = "eosio::chain::block_summary_object"
+	SectionNameTransactionObject           SectionName = "eosio::chain::transaction_object"
+	SectionNameGeneratedTransactionObject  SectionName = "eosio::chain::generated_transaction_object"
+	SectionNameCodeObject                  SectionName = "eosio::chain::code_object"
+	SectionNameContractTables              SectionName = "contract_tables"
+	SectionNamePermissionObject            SectionName = "eosio::chain::permission_object"
+	SectionNamePermissionLinkObject        SectionName = "eosio::chain::permission_link_object"
+	SectionNameResourceLimitsObject        SectionName = "eosio::chain::resource_limits::resource_limits_object"
+	SectionNameResourceUsageObject         SectionName = "eosio::chain::resource_limits::resource_usage_object"
+	SectionNameResourceLimitsStateObject   SectionName = "eosio::chain::resource_limits::resource_limits_state_object"
+	SectionNameResourceLimitsConfigObject  SectionName = "eosio::chain::resource_limits::resource_limits_config_object"
+	SectionNameGenesisState                SectionName = "eosio::chain::genesis_state"
+)
+
+func stringToSectionName(name string) SectionName {
+	switch name {
+	case "eosio::chain::chain_snapshot_header":
+		return SectionNameChainSnapshotHeader
+	case "eosio::chain::block_state":
+		return SectionNameBlockState
+	case "eosio::chain::account_object":
+		return SectionNameAccountObject
+	case "eosio::chain::account_metadata_object":
+		return SectionNameAccountMetadataObject
+	case "eosio::chain::account_ram_correction_object":
+		return SectionNameAccountRamCorrectionObject
+	case "eosio::chain::global_property_object":
+		return SectionNameGlobalPropertyObject
+	case "eosio::chain::protocol_state_object":
+		return SectionNameProtocolStateObject
+	case "eosio::chain::dynamic_global_property_object":
+		return SectionNameDynamicGlobalPropertyObject
+	case "eosio::chain::block_summary_object":
+		return SectionNameBlockSummaryObject
+	case "eosio::chain::transaction_object":
+		return SectionNameTransactionObject
+	case "eosio::chain::generated_transaction_object":
+		return SectionNameGeneratedTransactionObject
+	case "eosio::chain::code_object":
+		return SectionNameCodeObject
+	case "contract_tables":
+		return SectionNameContractTables
+	case "eosio::chain::permission_object":
+		return SectionNamePermissionObject
+	case "eosio::chain::permission_link_object":
+		return SectionNamePermissionLinkObject
+	case "eosio::chain::resource_limits::resource_limits_object":
+		return SectionNameResourceLimitsObject
+	case "eosio::chain::resource_limits::resource_usage_object":
+		return SectionNameResourceUsageObject
+	case "eosio::chain::resource_limits::resource_limits_state_object":
+		return SectionNameResourceLimitsStateObject
+	case "eosio::chain::resource_limits::resource_limits_config_object":
+		return SectionNameResourceLimitsConfigObject
+	case "eosio::chain::genesis_state":
+		return SectionNameGenesisState
+	default:
+		panic("unsupported section name: " + name)
+	}
+}
+
 type Section struct {
-	Name       string
+	Name       SectionName
 	Offset     uint64
 	Size       uint64 // This includes the section name and row count
 	BufferSize uint64 // This represents the bytes that are following the section header
@@ -20,48 +92,46 @@ type callbackFunc func(obj interface{}) error
 
 func (s *Section) Process(f callbackFunc) error {
 	switch s.Name {
-	case "eosio::chain::chain_snapshot_header":
+	case SectionNameChainSnapshotHeader:
 		return s.readChainSnapshotHeader(f)
-	case "eosio::chain::block_state":
+	case SectionNameBlockState:
 		return s.readBlockState(f)
-	case "eosio::chain::account_object":
+	case SectionNameAccountObject:
 		return s.readAccountObjects(f)
-	case "eosio::chain::account_metadata_object":
+	case SectionNameAccountMetadataObject:
 		return s.readAccountMetadataObjects(f)
-	case "eosio::chain::account_ram_correction_object":
+	case SectionNameAccountRamCorrectionObject:
 		return s.readAccountRAMCorrectionObject(f)
-	case "eosio::chain::global_property_object":
+	case SectionNameGlobalPropertyObject:
 		return s.readGlobalPropertyObject(f)
-	case "eosio::chain::protocol_state_object":
+	case SectionNameProtocolStateObject:
 		return s.readProtocolStateObject(f)
-	case "eosio::chain::dynamic_global_property_object":
+	case SectionNameDynamicGlobalPropertyObject:
 		return s.readDynamicGlobalPropertyObject(f)
-	case "eosio::chain::block_summary_object":
+	case SectionNameBlockSummaryObject:
 		return s.readBlockSummary(f)
-	case "eosio::chain::transaction_object":
+	case SectionNameTransactionObject:
 		return s.readTransactionObject(f)
-	case "eosio::chain::generated_transaction_object":
+	case SectionNameGeneratedTransactionObject:
 		return s.readGeneratedTransactionObject(f)
-	case "eosio::chain::code_object":
+	case SectionNameCodeObject:
 		return s.readCodeObject(f)
-	case "contract_tables":
+	case SectionNameContractTables:
 		return s.readContractTables(f)
-	case "eosio::chain::permission_object":
+	case SectionNamePermissionObject:
 		return s.readPermissionObject(f)
-	case "eosio::chain::permission_link_object":
+	case SectionNamePermissionLinkObject:
 		return s.readPermissionLinkObject(f)
-	case "eosio::chain::resource_limits::resource_limits_object":
+	case SectionNameResourceLimitsObject:
 		return s.readResourceLimitsObject(f)
-	case "eosio::chain::resource_limits::resource_usage_object":
+	case SectionNameResourceUsageObject:
 		return s.readResourceUsageObject(f)
-	case "eosio::chain::resource_limits::resource_limits_state_object":
+	case SectionNameResourceLimitsStateObject:
 		return s.readResourceLimitsStateObject(f)
-	case "eosio::chain::resource_limits::resource_limits_config_object":
+	case SectionNameResourceLimitsConfigObject:
 		return s.readResourceLimitsConfigObject(f)
-
-	case "eosio::chain::genesis_state":
+	case SectionNameGenesisState:
 		return s.readGenesisState(f)
-
 	default:
 		panic("unsupported section: " + s.Name)
 	}

snapshot/snapshot_test.go

@@ -4,50 +4,100 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"strings"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
 )
 
 func TestSnapshotRead(t *testing.T) {
-	// filename := "/tmp/0125111385-07750c59b24ed52d2dbf2048b67b58e9c9bd53ff5cc4550277718c1d5d800f73-snapshot.bin" // mainnet
-	// filename := "/tmp/0003212331-0031042b02b2cf711fee6e1e24da94101fa6c1ea9ece568d5f13232473429db1-snapshot.bin" // kylin
-	filename := os.Getenv("READ_SNAPSHOT_FILE")
-	if filename == "" || !fileExists(filename) {
-		t.Skipf("Environment varaible 'READ_SNAPSHOT_FILE' not set or value %q is not an exisiting file", filename)
-		return
+
+	//if os.Getenv("READ_SNAPSHOT_FILE") != "true" {
+	//	t.Skipf("Environment varaible 'READ_SNAPSHOT_FILE' not set to true")
+	//	return
+	//}
+
+	logger, _ := zap.NewDevelopment()
+	tests := []struct {
+		name     string
+		testFile string
+	}{
+		{name: "eos-local dev", testFile: "eos-jdev_0000000638.bin"},
+		{name: "eos-dev1", testFile: "eos-dev1_0004841949.bin"},
+		{name: "Battlefield - b8d703ed1", testFile: "battlefield-snapshot.bin"},
 	}
-	r, err := NewReader(filename)
-	fmt.Println("Filename", filename)
-	defer r.Close()
 
-	assert.NoError(t, err)
-	assert.Equal(t, r.Header.Version, uint32(1))
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			testFile := testData(test.testFile)
 
-	for {
-		section, err := r.Next()
-		if err == io.EOF {
-			break
-		}
-		assert.NoError(t, err)
-		fmt.Println("Section", section.Name, "rows", section.RowCount, "bytes", section.BufferSize, "offset", section.Offset)
+			if !fileExists(testFile) {
+				logger.Error("test file not found", zap.String("testfile", testFile))
+				return
+			}
 
-		if strings.Contains(section.Name, "contract") {
-			require.NoError(t, section.Process(func(o interface{}) error {
-				switch obj := o.(type) {
-				case *TableIDObject:
-					fmt.Println("Table ID", obj.Code, obj.Scope, obj.TableName)
-				case *KeyValueObject:
-					fmt.Println("KV", obj.PrimKey, obj.Value)
-				default:
-					fmt.Printf("Ignoring row %T\n", obj)
-				}
-				return nil
+			r, err := NewReader(testFile)
+			require.NoError(t, err)
+			defer r.Close()
+
+			assert.NoError(t, err)
+			assert.Equal(t, r.Header.Version, uint32(1))
 
-			}))
-		}
+			for {
+				section, err := r.Next()
+				if err == io.EOF {
+					break
+				}
+				assert.NoError(t, err)
+				logger.Info("new section",
+					zap.String("section_name", string(section.Name)),
+					zap.Uint64("row_count", section.RowCount),
+					zap.Uint64("bytes_count", section.BufferSize),
+					zap.Uint64("bytes_count", section.Offset),
+				)
+				switch section.Name {
+				case SectionNameAccountObject:
+					require.NoError(t, section.Process(func(o interface{}) error {
+						acc, ok := o.(AccountObject)
+						if !ok {
+							return fmt.Errorf("process account object: unexpected object type: %T", o)
+						}
+						logger.Info("new account object",
+							zap.String("name", string(acc.Name)),
+						)
+						return nil
+					}))
+				case SectionNameContractTables:
+					var currentTable *TableIDObject
+					require.NoError(t, section.Process(func(o interface{}) error {
+						switch obj := o.(type) {
+						case *TableIDObject:
+							logger.Info("Table ID", zap.Reflect("table_id", obj))
+							currentTable = obj
+						case *KeyValueObject:
+							if currentTable.Count < 20 {
+								logger.Info("Key Value Object", zap.Reflect("kv", obj))
+							}
+						case *Index64Object:
+							logger.Info("Index64Object", zap.Reflect("index_64_object", obj))
+						case *Index128Object:
+							logger.Info("Index128Object", zap.Reflect("index_128_object", obj))
+						case *Index256Object:
+							logger.Info("Index256Object", zap.Reflect("index_256_object", obj))
+						case *IndexDoubleObject:
+							logger.Info("IndexDoubleObject", zap.Reflect("index_double_object", obj))
+						case *IndexLongDoubleObject:
+							logger.Info("IndexLongDoubleObject", zap.Reflect("index_long_object", obj))
+						default:
+							fmt.Printf("Ignoring row %T\n", obj)
+						}
+						return nil
+					}))
+				}
+			}
+		})
 	}
 }
 
@@ -63,3 +113,7 @@ func fileExists(path string) bool {
 
 	return !info.IsDir()
 }
+
+func testData(filename string) string {
+	return filepath.Join("test-data", filename)
+}

snapshot/typesv3.go

@@ -14,12 +14,10 @@ type TableIDObject struct {
 	Scope     string
 	TableName string
 	Payer     string
-	Count     uint32
+	Count     uint32 // represents the number of rows & indices for a given table
 }
 
 type ContractRow struct {
-	TableID *TableIDObject
-
 	PrimKey string
 	Payer   string
 }
@@ -131,7 +129,6 @@ func (section *Section) readContractTables(f callbackFunc) error {
 				}
 
 				contractRow := ContractRow{
-					TableID: t,
 					PrimKey: eos.NameToString(binary.LittleEndian.Uint64(head[0:8])),
 					Payer:   eos.NameToString(binary.LittleEndian.Uint64(head[8:16])),
 				}