feat: add optional verification of input and output file with separate csv file

jonastheis · jonastheis · commit dcdd6f05377f · 2025-05-22T09:07:30.000+02:00
diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go
@@ -2,15 +2,20 @@ package cmd
 
 import (
 	"bufio"
+	"bytes"
 	"crypto/sha256"
 	"encoding/binary"
 	"fmt"
 	"io"
 	"log"
 	"os"
+	"strconv"
+	"strings"
 
 	"github.com/spf13/cobra"
 
+	"github.com/scroll-tech/go-ethereum/common"
+
 	"github.com/scroll-tech/go-ethereum/export-headers-toolkit/types"
 )
 
@@ -38,33 +43,32 @@ The binary layout of the deduplicated file is as follows:
 		if err != nil {
 			log.Fatalf("Error reading output flag: %v", err)
 		}
+		verifyFile, err := cmd.Flags().GetString("verify")
+		if err != nil {
+			log.Fatalf("Error reading verify flag: %v", err)
+		}
+
+		if verifyFile != "" {
+			verifyInputFile(verifyFile, inputFile)
+		}
 
 		_, seenVanity, _ := runAnalysis(inputFile)
 		runDedup(inputFile, outputFile, seenVanity)
-		runSHA256(outputFile)
-	},
-}
 
-func runSHA256(outputFile string) {
-	f, err := os.Open(outputFile)
-	defer f.Close()
-	if err != nil {
-		log.Fatalf("Error opening file: %v", err)
-	}
-
-	h := sha256.New()
-	if _, err = io.Copy(h, f); err != nil {
-		log.Fatalf("Error hashing file: %v", err)
-	}
+		if verifyFile != "" {
+			verifyOutputFile(verifyFile, outputFile)
+		}
 
-	fmt.Printf("Deduplicated headers written to %s with sha256 checksum: %x\n", outputFile, h.Sum(nil))
+		runSHA256(outputFile)
+	},
 }
 
 func init() {
 	rootCmd.AddCommand(dedupCmd)
 
 	dedupCmd.Flags().String("input", "headers.bin", "headers file")
 	dedupCmd.Flags().String("output", "headers-dedup.bin", "deduplicated, binary formatted file")
+	dedupCmd.Flags().String("verify", "", "verify the input and output files with the given .csv file")
 }
 
 func runAnalysis(inputFile string) (seenDifficulty map[uint64]int, seenVanity map[[32]byte]bool, seenSealLen map[int]int) {
@@ -118,6 +122,21 @@ func runDedup(inputFile, outputFile string, seenVanity map[[32]byte]bool) {
 	})
 }
 
+func runSHA256(outputFile string) {
+	f, err := os.Open(outputFile)
+	defer f.Close()
+	if err != nil {
+		log.Fatalf("Error opening file: %v", err)
+	}
+
+	h := sha256.New()
+	if _, err = io.Copy(h, f); err != nil {
+		log.Fatalf("Error hashing file: %v", err)
+	}
+
+	fmt.Printf("Deduplicated headers written to %s with sha256 checksum: %x\n", outputFile, h.Sum(nil))
+}
+
 type headerReader struct {
 	file       *os.File
 	reader     *bufio.Reader
@@ -175,3 +194,103 @@ func (h *headerReader) read(callback func(header *types.Header)) {
 func (h *headerReader) close() {
 	h.file.Close()
 }
+
+type csvHeaderReader struct {
+	file   *os.File
+	reader *bufio.Reader
+}
+
+func newCSVHeaderReader(verifyFile string) *csvHeaderReader {
+	f, err := os.Open(verifyFile)
+	if err != nil {
+		log.Fatalf("Error opening verify file: %v", err)
+	}
+
+	h := &csvHeaderReader{
+		file:   f,
+		reader: bufio.NewReader(f),
+	}
+
+	return h
+}
+
+func (h *csvHeaderReader) readNext() *types.Header {
+	line, err := h.reader.ReadString('\n')
+	if err != nil {
+		if err == io.EOF {
+			return nil
+		}
+		log.Fatalf("Error reading line: %v", err)
+	}
+
+	s := strings.Split(line, ",")
+	extraString := strings.Split(s[2], "\n")
+
+	num, err := strconv.ParseUint(s[0], 10, 64)
+	if err != nil {
+		log.Fatalf("Error parsing block number: %v", err)
+	}
+	difficulty, err := strconv.ParseUint(s[1], 10, 64)
+	if err != nil {
+		log.Fatalf("Error parsing difficulty: %v", err)
+	}
+	extra := common.FromHex(extraString[0])
+
+	header := types.NewHeader(num, difficulty, extra)
+	return header
+}
+
+func (h *csvHeaderReader) close() {
+	h.file.Close()
+}
+
+func verifyInputFile(verifyFile, inputFile string) {
+	csvReader := newCSVHeaderReader(verifyFile)
+	defer csvReader.close()
+
+	binaryReader := newHeaderReader(inputFile)
+	defer binaryReader.close()
+
+	binaryReader.read(func(header *types.Header) {
+		csvHeader := csvReader.readNext()
+
+		if !csvHeader.Equal(header) {
+			log.Fatalf("Header mismatch: %v != %v", csvHeader, header)
+		}
+	})
+
+	log.Printf("All headers match in %s and %s\n", verifyFile, inputFile)
+}
+
+func verifyOutputFile(verifyFile, outputFile string) {
+	csvReader := newCSVHeaderReader(verifyFile)
+	defer csvReader.close()
+
+	dedupReader, err := NewReader(outputFile)
+	if err != nil {
+		log.Fatalf("Error opening dedup file: %v", err)
+	}
+	defer dedupReader.Close()
+
+	for {
+		header := csvReader.readNext()
+		if header == nil {
+			if _, _, err = dedupReader.ReadNext(); err == nil {
+				log.Fatalf("Expected EOF, got more headers")
+			}
+			break
+		}
+
+		difficulty, extraData, err := dedupReader.Read(header.Number)
+		if err != nil {
+			log.Fatalf("Error reading header: %v", err)
+		}
+
+		if header.Difficulty != difficulty {
+			log.Fatalf("Difficulty mismatch: headerNum %d: %d != %d", header.Number, header.Difficulty, difficulty)
+		}
+		if !bytes.Equal(header.ExtraData, extraData) {
+			log.Fatalf("ExtraData mismatch: headerNum %d: %x != %x", header.Number, header.ExtraData, extraData)
+		}
+	}
+}
diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go
@@ -0,0 +1,120 @@
+package cmd
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+)
+
+// TODO: instead of duplicating this file, missing_header_fields.Reader should be used in toolkit
+
+type missingHeader struct {
+	headerNum  uint64
+	difficulty uint64
+	extraData  []byte
+}
+
+type Reader struct {
+	file           *os.File
+	reader         *bufio.Reader
+	sortedVanities map[int][32]byte
+	lastReadHeader *missingHeader
+}
+
+func NewReader(filePath string) (*Reader, error) {
+	f, err := os.Open(filePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file: %v", err)
+	}
+
+	r := &Reader{
+		file:   f,
+		reader: bufio.NewReader(f),
+	}
+
+	// read the count of unique vanities
+	vanityCount, err := r.reader.ReadByte()
+	if err != nil {
+		return nil, err
+	}
+
+	// read the unique vanities
+	r.sortedVanities = make(map[int][32]byte)
+	for i := uint8(0); i < vanityCount; i++ {
+		var vanity [32]byte
+		if _, err = r.reader.Read(vanity[:]); err != nil {
+			return nil, err
+		}
+		r.sortedVanities[int(i)] = vanity
+	}
+
+	return r, nil
+}
+
+func (r *Reader) Read(headerNum uint64) (difficulty uint64, extraData []byte, err error) {
+	if r.lastReadHeader == nil {
+		if _, _, err = r.ReadNext(); err != nil {
+			return 0, nil, err
+		}
+	}
+
+	if headerNum > r.lastReadHeader.headerNum {
+		// skip the headers until the requested header number
+		for i := r.lastReadHeader.headerNum; i < headerNum; i++ {
+			if _, _, err = r.ReadNext(); err != nil {
+				return 0, nil, err
+			}
+		}
+	}
+
+	if headerNum == r.lastReadHeader.headerNum {
+		return r.lastReadHeader.difficulty, r.lastReadHeader.extraData, nil
+	}
+
+	// headerNum < r.lastReadHeader.headerNum is not supported
+	return 0, nil, fmt.Errorf("requested header %d below last read header number %d", headerNum, r.lastReadHeader.headerNum)
+}
+
+func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) {
+	// read the bitmask
+	bitmaskByte, err := r.reader.ReadByte()
+	if err != nil {
+		return 0, nil, fmt.Errorf("failed to read bitmask: %v", err)
+	}
+
+	bits := newBitMaskFromByte(bitmaskByte)
+
+	seal := make([]byte, bits.sealLen())
+
+	if _, err = io.ReadFull(r.reader, seal); err != nil {
+		return 0, nil, fmt.Errorf("failed to read seal: %v", err)
+	}
+
+	// construct the extraData field
+	vanity := r.sortedVanities[bits.vanityIndex()]
+	var b bytes.Buffer
+	b.Write(vanity[:])
+	b.Write(seal)
+
+	// we don't have the header number, so we'll just increment the last read header number
+	// we assume that the headers are written in order, starting from 0
+	if r.lastReadHeader == nil {
+		r.lastReadHeader = &missingHeader{
+			headerNum:  0,
+			difficulty: uint64(bits.difficulty()),
+			extraData:  b.Bytes(),
+		}
+	} else {
+		r.lastReadHeader.headerNum++
+		r.lastReadHeader.difficulty = uint64(bits.difficulty())
+		r.lastReadHeader.extraData = b.Bytes()
+	}
+
+	return difficulty, b.Bytes(), nil
+}
+
+func (r *Reader) Close() error {
+	return r.file.Close()
+}
diff --git a/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go b/rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go
@@ -115,6 +115,10 @@ type bitMask struct {
 	b uint8
 }
 
+func newBitMaskFromByte(b uint8) bitMask {
+	return bitMask{b}
+}
+
 func newBitMask(vanityIndex int, difficulty int, sealLen int) bitMask {
 	b := uint8(0)
 
diff --git a/rollup/missing_header_fields/export-headers-toolkit/types/header.go b/rollup/missing_header_fields/export-headers-toolkit/types/header.go
@@ -3,6 +3,8 @@ package types
 import (
 	"encoding/binary"
 	"fmt"
+
+	"github.com/scroll-tech/go-ethereum/common"
 )
 
 const HeaderSizeSerialized = 2
@@ -23,7 +25,25 @@ func NewHeader(number, difficulty uint64, extraData []byte) *Header {
 }
 
 func (h *Header) String() string {
-	return fmt.Sprintf("%d,%d,0x%x\n", h.Number, h.Difficulty, h.ExtraData)
+	return fmt.Sprintf("%d,%d,%s\n", h.Number, h.Difficulty, common.Bytes2Hex(h.ExtraData))
+}
+
+func (h *Header) Equal(other *Header) bool {
+	if h.Number != other.Number {
+		return false
+	}
+	if h.Difficulty != other.Difficulty {
+		return false
+	}
+	if len(h.ExtraData) != len(other.ExtraData) {
+		return false
+	}
+	for i, b := range h.ExtraData {
+		if b != other.ExtraData[i] {
+			return false
+		}
+	}
+	return true
 }
 
 // Bytes returns the byte representation of the header including the initial 2 bytes for the size.

Original file line number	Diff line number	Diff line change
`@@ -115,6 +115,10 @@ type bitMask struct {`
`115`	`115`	`b uint8`
`116`	`116`	`}`
`117`	`117`
	`118`	`+func newBitMaskFromByte(b uint8) bitMask {`
	`119`	`+ return bitMask{b}`
	`120`	`+}`
	`121`	`+`
`118`	`122`	`func newBitMask(vanityIndex int, difficulty int, sealLen int) bitMask {`
`119`	`123`	`b := uint8(0)`
`120`	`124`