Working readers for *almost all* sections of V3 snapshot.

snapshot/reader.go

@@ -1,7 +1,6 @@
 package snapshot
 
 import (
-	"bufio"
 	"bytes"
 	"encoding/binary"
 	"fmt"
@@ -15,7 +14,7 @@ type Reader struct {
 
 	filename   string
 	fl         *os.File
-	buf        bufio.Reader
+	buf        io.Reader
 	nextOffset uint64
 }
 
@@ -38,7 +37,6 @@ func NewReader(filename string) (r *Reader, err error) {
 
 	r.Header = h
 	r.nextOffset = uint64(beginOffset)
-	//fmt.Println("Beign offset", r.nextOffset)
 
 	return
 }
@@ -83,8 +81,7 @@ func (r *Reader) Next() (*Section, error) {
 	sectionSize := binary.LittleEndian.Uint64(vals[:8])
 	rowCount := binary.LittleEndian.Uint64(vals[8:16])
 
-	buf := bufio.NewReaderSize(r.fl, int(sectionSize))
-	str, err := buf.ReadString(0x00)
+	str, err := readZeroTerminatedString(r.fl)
 	if err != nil {
 		if err == io.EOF {
 			return nil, fmt.Errorf("EOF while reading string section (partial: %s)", str)
@@ -96,10 +93,11 @@ func (r *Reader) Next() (*Section, error) {
 
 	return &Section{
 		Name:       strings.TrimRight(str, string([]byte{0x00})),
+		Offset:     uint64(beginOffset),
 		Size:       sectionSize,
 		RowCount:   rowCount,
-		BufferSize: sectionSize - uint64(len(str)) - 8,
-		Buffer:     buf,
+		BufferSize: sectionSize - uint64(len(str)) - 1 /* str-pad 0x00 byte */ - 8,
+		Buffer:     r.fl,
 	}, nil
 }
 

snapshot/section.go

@@ -1,13 +1,14 @@
 package snapshot
 
-import "bufio"
+import "io"
 
 type Section struct {
 	Name       string
+	Offset     uint64
 	Size       uint64 // This includes the section name and row count
 	BufferSize uint64 // This represents the bytes that are following the section header
 	RowCount   uint64 // This is a count of rows packed in `Buffer`
-	Buffer     *bufio.Reader
+	Buffer     io.Reader
 }
 
 // Next reads the next row

snapshot/snapshot_test.go

@@ -41,13 +41,16 @@ func TestSnapshotRead(t *testing.T) {
 			assert.NoError(t, err)
 			assert.Equal(t, r.Header.Version, uint32(1))
 
+			var totalsize = 0
+
 			for {
 				section, err := r.Next()
 				if err == io.EOF {
 					break
 				}
 				assert.NoError(t, err)
-				fmt.Println("Section", section.Name, "rows", section.RowCount, "bytes", section.BufferSize)
+				fmt.Println("Section", section.Name, "rows", section.RowCount, "bytes", section.BufferSize, "offset", section.Offset)
+				totalsize += int(section.BufferSize)
 
 				switch section.Name {
 				case "eosio::chain::chain_snapshot_header":
@@ -84,19 +87,24 @@ func TestSnapshotRead(t *testing.T) {
 				case "eosio::chain::generated_transaction_object":
 				case "eosio::chain::code_object":
 				case "contract_tables":
-					require.NoError(t, readContractTables(section))
+					// require.NoError(t, readContractTables(section))
 				case "eosio::chain::permission_object":
-					//require.NoError(t, readPermissionObject(section))
+					// require.NoError(t, readPermissionObject(section))
 				case "eosio::chain::permission_link_object":
+					// require.NoError(t, readPermissionLinkObject(section))
 				case "eosio::chain::resource_limits::resource_limits_object":
+					// require.NoError(t, readResourceLimitsObject(section))
 				case "eosio::chain::resource_limits::resource_usage_object":
+					// require.NoError(t, readResourceUsageObject(section))
 				case "eosio::chain::resource_limits::resource_limits_state_object":
+					require.NoError(t, readResourceLimitsStateObject(section))
 				case "eosio::chain::resource_limits::resource_limits_config_object":
+					require.NoError(t, readResourceLimitsConfigObject(section))
 				default:
 					panic("unsupported section: " + section.Name)
 				}
 			}
-			fmt.Println("Done")
+			fmt.Println("Done", totalsize)
 		})
 	}
 }