Skip to content

Commit

Permalink
remove BlockTime from oneBlockFile, use longer ID
Browse files Browse the repository at this point in the history 
fixes: streamingfast/merger#18
  • Loading branch information
@sduchesneau
sduchesneau committed Jul 18, 2022
1 parent e141dd6 commit ac92791
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 104 deletions.
77 changes: 25 additions & 52 deletions oneblockfile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"io/ioutil"
"strconv"
"strings"
"time"

"github.com/streamingfast/dstore"
)
Expand All @@ -32,28 +31,22 @@ type OneBlockDownloaderFunc = func(ctx context.Context, oneBlockFile *OneBlockFi
type OneBlockFile struct {
CanonicalName string
Filenames map[string]bool
BlockTime time.Time
ID string
Num uint64
InnerLibNum *uint64 //never use this field directly
LibNum uint64
PreviousID string
MemoizeData []byte
Deleted bool
}

// ToBstreamBlock is used to create a dummy "empty" block to use in a ForkableHandler
func (f *OneBlockFile) ToBstreamBlock() *Block {
blk := &Block{
return &Block{
Id: f.ID,
Number: f.Num,
PreviousId: f.PreviousID,
Timestamp: f.BlockTime,
LibNum: f.LibNum,
}

if f.InnerLibNum != nil {
blk.LibNum = *f.InnerLibNum
}
return blk
}

func (f *OneBlockFile) String() string {
Expand All @@ -62,7 +55,7 @@ func (f *OneBlockFile) String() string {

func NewOneBlockFile(fileName string) (*OneBlockFile, error) {
_ = &Block{}
blockNum, blockTime, blockID, previousBlockID, libNum, canonicalName, err := ParseFilename(fileName)
blockNum, blockID, previousBlockID, libNum, canonicalName, err := ParseFilename(fileName)
if err != nil {
return nil, err
}
Expand All @@ -71,11 +64,10 @@ func NewOneBlockFile(fileName string) (*OneBlockFile, error) {
Filenames: map[string]bool{
fileName: true,
},
BlockTime: blockTime,
ID: blockID,
Num: blockNum,
PreviousID: previousBlockID,
InnerLibNum: libNum,
ID: blockID,
Num: blockNum,
PreviousID: previousBlockID,
LibNum: libNum,
}, nil
}

Expand All @@ -98,49 +90,33 @@ func (f *OneBlockFile) Data(ctx context.Context, oneBlockDownloader OneBlockDown
return f.MemoizeData, nil
}

func (f *OneBlockFile) LibNum() uint64 {
if f.InnerLibNum == nil {
panic("one block file lib num not set")
}
return *f.InnerLibNum
}

// ParseFilename parses file names formatted like:
// * 0000000101-20170701T122141.5-dbda3f44-24a07267-100-mindread1
// * 0000000101-20170701T122141.5-dbda3f44-24a07267-100-mindread2
// * 0000000101-dbda3f44afee24dd-24a072678473e4ad-100-mindread1
// * 0000000101-dbda3f44afee24dd-24a072678473e4ad-100-mindread2

func ParseFilename(filename string) (blockNum uint64, blockTime time.Time, blockIDSuffix string, previousBlockIDSuffix string, libNum *uint64, canonicalName string, err error) {
func ParseFilename(filename string) (blockNum uint64, blockIDSuffix string, previousBlockIDSuffix string, libNum uint64, canonicalName string, err error) {
parts := strings.Split(filename, "-")
if len(parts) < 4 || len(parts) > 6 {
if len(parts) != 5 {
err = fmt.Errorf("wrong filename format: %q", filename)
return
}

blockNumVal, err := strconv.ParseUint(parts[0], 10, 32)
if err != nil {
err = fmt.Errorf("failed parsing %q: %s", parts[0], err)
blockNumVal, parseErr := strconv.ParseUint(parts[0], 10, 32)
if parseErr != nil {
err = fmt.Errorf("failed parsing %q: %s", parts[0], parseErr)
return
}
blockNum = blockNumVal

blockTime, err = time.Parse("20060102T150405.999999", parts[1])
if err != nil {
err = fmt.Errorf("failed parsing %q: %s", parts[1], err)
return
}

blockIDSuffix = parts[2]
previousBlockIDSuffix = parts[3]
blockIDSuffix = parts[1]
previousBlockIDSuffix = parts[2]
canonicalName = filename
if len(parts) == 6 {
libNumVal, parseErr := strconv.ParseUint(parts[4], 10, 32)
if parseErr != nil {
err = fmt.Errorf("failed parsing lib num %q: %s", parts[4], parseErr)
return
}
libNum = &libNumVal
canonicalName = strings.Join(parts[0:5], "-")
libNum, parseErr = strconv.ParseUint(parts[3], 10, 32)
if parseErr != nil {
err = fmt.Errorf("failed parsing lib num %q: %s", parts[4], parseErr)
return
}
canonicalName = strings.Join(parts[0:4], "-")

return
}
Expand All @@ -150,20 +126,17 @@ func BlockFileName(block *Block) string {
}

func TruncateBlockID(in string) string {
if len(in) <= 8 {
if len(in) <= 16 {
return in
}
return in[len(in)-8:]
return in[len(in)-16:]
}

func BlockFileNameWithSuffix(block *Block, suffix string) string {
blockTime := block.Time()
blockTimeString := fmt.Sprintf("%s.%01d", blockTime.Format("20060102T150405"), blockTime.Nanosecond()/100000000)

blockID := TruncateBlockID(block.ID())
previousID := TruncateBlockID(block.PreviousID())

return fmt.Sprintf("%010d-%s-%s-%s-%d-%s", block.Num(), blockTimeString, blockID, previousID, block.LibNum, suffix)
return fmt.Sprintf("%010d-%s-%s-%d-%s", block.Num(), blockID, previousID, block.LibNum, suffix)
}

func OneBlockDownloaderFromStore(blocksStore dstore.Store) OneBlockDownloaderFunc {
Expand Down
70 changes: 18 additions & 52 deletions oneblockfile_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import (
"fmt"
"strings"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand All @@ -31,63 +30,54 @@ func TestOneBlockFile_MustNewOneBlockFile_Panics(t *testing.T) {
}

func TestOneBlockFile_MustNewOneBlockFile(t *testing.T) {
name := "0000000100-20210728T105016.01-00000100a-00000099a-90-suffix"
name := "0000000100-0000000000000100a-0000000000000099a-90-suffix"
obf := MustNewOneBlockFile(name)
require.IsType(t, OneBlockFile{}, *obf)
require.Equal(t, obf.CanonicalName, strings.Split(name, "-suffix")[0])
}

func TestOneBlockFile_ParseFilename(t *testing.T) {
lib := func(num uint64) *uint64 { lib := num; return &lib }
tests := []struct {
name string
filename string
expectBlockNum uint64
expectBlockTime time.Time
expectBlockIDSuffix string
expectPreviousBlockIDSuffix string
expectCanonicalName string
expectError error
expectLibNum *uint64
expectLibNum uint64
}{
{
name: "invalid",
filename: "invalid-filename",
expectError: fmt.Errorf("wrong filename format: \"invalid-filename\""),
},
{
name: "without lib",
filename: "0000000100-20170701T122141.0-24a07267-e5914b39",
expectBlockNum: 100,
expectLibNum: nil,
expectBlockTime: mustParseTime("20170701T122141.0"),
expectBlockIDSuffix: "24a07267",
expectPreviousBlockIDSuffix: "e5914b39",
expectCanonicalName: "0000000100-20170701T122141.0-24a07267-e5914b39",
name: "without lib",
filename: "0000000100-aaaabbbb24a07267-ccccdddde5914b39",
expectError: fmt.Errorf("wrong filename format: \"0000000100-aaaabbbb24a07267-ccccdddde5914b39\""),
},
{
name: "with suffix",
filename: "0000000100-20170701T122141.0-24a07267-e5914b39-90-mind1",
filename: "0000000100-aaaabbbb24a07267-ccccdddde5914b39-90-mind1",
expectBlockNum: 100,
expectLibNum: lib(90),
expectBlockTime: mustParseTime("20170701T122141.0"),
expectBlockIDSuffix: "24a07267",
expectPreviousBlockIDSuffix: "e5914b39",
expectCanonicalName: "0000000100-20170701T122141.0-24a07267-e5914b39-90",
expectLibNum: 90,
expectBlockIDSuffix: "aaaabbbb24a07267",
expectPreviousBlockIDSuffix: "ccccdddde5914b39",
expectCanonicalName: "0000000100-aaaabbbb24a07267-ccccdddde5914b39-90",
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
blkNum, blkTime, blkID, prevBlkID, libNum, name, err := ParseFilename(test.filename)
blkNum, blkID, prevBlkID, libNum, name, err := ParseFilename(test.filename)
if test.expectError != nil {
require.Equal(t, err, test.expectError)
return
}
require.Nil(t, err)
assert.Equal(t, test.expectBlockNum, blkNum)
assert.Equal(t, test.expectLibNum, libNum)
assert.Equal(t, test.expectBlockTime, blkTime)
assert.Equal(t, test.expectBlockIDSuffix, blkID)
assert.Equal(t, test.expectPreviousBlockIDSuffix, prevBlkID)
assert.Equal(t, test.expectCanonicalName, name)
Expand All @@ -96,46 +86,22 @@ func TestOneBlockFile_ParseFilename(t *testing.T) {
}

func TestOneBlockFile_ParseFilename_InvalidBlockNum(t *testing.T) {
name := "0000000FFF-20170701T122141.0-24a07267-e5914b39"
_, _, _, _, _, _, err := ParseFilename(name)
require.Error(t, err)
}

func TestOneBlockFile_ParseFilename_InvalidBlockTime(t *testing.T) {
name := "0000000100-20ABCDE1T122141.0-24a07267-e5914b39"
_, _, _, _, _, _, err := ParseFilename(name)
name := "0000000FFF-24a07267aaaaeeee-e5914b39bbbbffff-suffix"
_, _, _, _, _, err := ParseFilename(name)
require.Error(t, err)
}

func TestOneBlockFile_ParseFilename_InvalidLibNum(t *testing.T) {
name := "0000000100-20170701T122141.0-24a07267-e5914b39-FFFF-suffix"
_, _, _, _, _, _, err := ParseFilename(name)
name := "0000000100-24a07267aaaaeeee-e5914b39bbbb4444-FFFF-suffix"
_, _, _, _, _, err := ParseFilename(name)
require.Error(t, err)
}

func mustParseTime(in string) time.Time {
t, err := time.Parse("20060102T150405.999999", in)
if err != nil {
panic("invalid parsetime")
}
return t
}

func TestOneBlockFile_InnerLibNumPanic(t *testing.T) {
name := "0000000100-20210728T105016.01-00000100a-00000099a-90-suffix"
obf := MustNewOneBlockFile(name)

defer func() { recover() }()
obf.InnerLibNum = nil
_ = obf.LibNum()
}

func newBstreamBlock() *Block {
block := Block{
Id: "longerthan8",
Id: "muchlongerthan16chars",
Number: uint64(0),
PreviousId: "longerthan8too",
Timestamp: mustParseTime("20170701T122141.0"),
PreviousId: "muchlongerthan16charsalso",
LibNum: uint64(0),
}

Expand All @@ -145,5 +111,5 @@ func newBstreamBlock() *Block {
func TestOneBlockFile_BlockFileName(t *testing.T) {
block := newBstreamBlock()
bfn := BlockFileName(block)
require.Equal(t, bfn, "0000000000-20170701T122141.0-gerthan8-than8too-0-generated")
require.Equal(t, bfn, "0000000000-ongerthan16chars-rthan16charsalso-0-generated")
}

0 comments on commit ac92791

Please sign in to comment.