feat: Txs To Shares Roundtrip (#889)

Adds ability to:
- Convert `TxWithISRs` to Shares
- Convert Shares to byte array (that will be posted on celestia)
- Convert above byte array back to Shares
- Convert Shares back to `TxWithISRs`
- Parse Out of Context Shares

Adds relevant roundtrip tests

Resolves #881 
Resolves #934 
Resolves #886 
Resolves #925 

Note: All shares are written and interpreted as compact shares so they
contain reserved bytes (see
https://celestiaorg.github.io/celestia-app/specs/data_structures.html#compact-share)

Related PR in `celestia-app`:
celestiaorg/celestia-app#1770

third_party/celestia-app/shares/compact_shares_test.go

@@ -6,7 +6,6 @@ import (
  "testing"
  "time"
 
- coretypes "github.com/cometbft/cometbft/types"
  "github.com/stretchr/testify/assert"
  "github.com/stretchr/testify/require"
 
@@ -15,24 +14,6 @@ import (
  "github.com/rollkit/rollkit/third_party/celestia-app/testfactory"
 )
 
-func SplitTxs(txs coretypes.Txs) (txShares []Share, err error) {
- txWriter := NewCompactShareSplitter(appns.TxNamespace, appconsts.ShareVersionZero)
-
- for _, tx := range txs {
- err = txWriter.WriteTx(tx)
- if err != nil {
- return nil, err
- }
- }
-
- txShares, _, err = txWriter.Export(0)
- if err != nil {
- return nil, err
- }
-
- return txShares, nil
-}
-
 func TestCompactShareSplitter(t *testing.T) {
  // note that this test is mainly for debugging purposes, the main round trip
  // tests occur in TestMerge and Test_processCompactShares
@@ -131,6 +112,29 @@ func Test_processCompactShares(t *testing.T) {
  }
 }
 
+func TestAllSplit(t *testing.T) {
+ txs := testfactory.GenerateRandomlySizedTxs(1000, 150)
+ txShares, err := SplitTxs(txs)
+ require.NoError(t, err)
+ resTxs, err := ParseTxs(txShares)
+ require.NoError(t, err)
+ assert.Equal(t, resTxs, txs)
+}
+
+func TestParseRandomOutOfContextShares(t *testing.T) {
+ txs := testfactory.GenerateRandomlySizedTxs(1000, 150)
+ txShares, err := SplitTxs(txs)
+ require.NoError(t, err)
+
+ for i := 0; i < 1000; i++ {
+ start, length := testfactory.GetRandomSubSlice(len(txShares))
+ randomRange := NewRange(start, start+length)
+ resTxs, err := ParseTxs(txShares[randomRange.Start:randomRange.End])
+ require.NoError(t, err)
+ assert.True(t, testfactory.CheckSubArray(txs, resTxs))
+ }
+}
+
 func TestCompactShareContainsInfoByte(t *testing.T) {
  css := NewCompactShareSplitter(appns.TxNamespace, appconsts.ShareVersionZero)
  txs := testfactory.GenerateRandomTxs(1, appconsts.ContinuationCompactShareContentSize/4)
@@ -197,10 +201,6 @@ func Test_parseCompactSharesErrors(t *testing.T) {
  }
 
  testCases := []testCase{
- {
- "share with start indicator false",
- txShares[1:], // set the first share to the second share which has the start indicator set to false
- },
  {
  "share with unsupported share version",
  []Share{*shareWithUnsupportedShareVersion},

third_party/celestia-app/shares/parse_compact_shares.go

@@ -1,6 +1,12 @@
 package shares
 
-import "errors"
+import (
+ "github.com/rollkit/rollkit/third_party/celestia-app/appconsts"
+)
+
+func ParseCompactShares(shares []Share) (data [][]byte, err error) {
+ return parseCompactShares(shares, appconsts.SupportedShareVersions)
+}
 
 // parseCompactShares returns data (transactions or intermediate state roots
 // based on the contents of rawShares and supportedShareVersions. If rawShares
@@ -13,14 +19,6 @@ func parseCompactShares(shares []Share, supportedShareVersions []uint8) (data []
  return nil, nil
  }
 
- seqStart, err := shares[0].IsSequenceStart()
- if err != nil {
- return nil, err
- }
- if !seqStart {
- return nil, errors.New("first share is not the start of a sequence")
- }
-
  err = validateShareVersions(shares, supportedShareVersions)
  if err != nil {
  return nil, err
@@ -61,19 +59,32 @@ func parseRawData(rawData []byte) (units [][]byte, err error) {
  if err != nil {
  return nil, err
  }
+ // the rest of raw data is padding
  if unitLen == 0 {
  return units, nil
  }
+ // the rest of actual data contains only part of the next transaction so
+ // we stop parsing raw data
+ if unitLen > uint64(len(actualData)) {
+ return units, nil
+ }
  rawData = actualData[unitLen:]
  units = append(units, actualData[:unitLen])
  }
 }
 
-// extractRawData returns the raw data contained in the shares. The raw data does
-// not contain the namespace ID, info byte, sequence length, or reserved bytes.
+// extractRawData returns the raw data representing complete transactions
+// contained in the shares. The raw data does not contain the namespace, info
+// byte, sequence length, or reserved bytes. Starts reading raw data based on
+// the reserved bytes in the first share.
 func extractRawData(shares []Share) (rawData []byte, err error) {
  for i := 0; i < len(shares); i++ {
- raw, err := shares[i].RawData()
+ var raw []byte
+ if i == 0 {
+ raw, err = shares[i].RawDataUsingReserved()
+ } else {
+ raw, err = shares[i].RawData()
+ }
  if err != nil {
  return nil, err
  }

third_party/celestia-app/shares/range.go

@@ -0,0 +1,26 @@
+package shares
+
+// Range is an end exclusive set of share indexes.
+type Range struct {
+ // Start is the index of the first share occupied by this range.
+ Start int
+ // End is the next index after the last share occupied by this range.
+ End int
+}
+
+func NewRange(start, end int) Range {
+ return Range{Start: start, End: end}
+}
+
+func EmptyRange() Range {
+ return Range{Start: 0, End: 0}
+}
+
+func (r Range) IsEmpty() bool {
+ return r.Start == 0 && r.End == 0
+}
+
+func (r *Range) Add(value int) {
+ r.Start += value
+ r.End += value
+}

third_party/celestia-app/shares/share_builder.go

@@ -123,12 +123,12 @@ func (b *Builder) indexOfInfoBytes() int {
  return appconsts.NamespaceSize
 }
 
-// MaybeWriteReservedBytes will be a no-op if the reserved bytes
+// MaybeWriteReservedBytes will be a no-op for a compact share or if the reserved bytes
 // have already been populated. If the reserved bytes are empty, it will write
 // the location of the next unit of data to the reserved bytes.
 func (b *Builder) MaybeWriteReservedBytes() error {
  if !b.isCompactShare {
- return errors.New("this is not a compact share")
+ return nil
  }
 
  empty, err := b.isEmptyReservedBytes()

third_party/celestia-app/shares/shares.go

@@ -161,6 +161,29 @@ func (s *Share) ToBytes() []byte {
  return s.data
 }
 
+// RawDataWithReserved returns the raw share data including the reserved bytes. The raw share data does not contain the namespace ID, info byte, or sequence length.
+func (s *Share) RawDataWithReserved() (rawData []byte, err error) {
+ if len(s.data) < s.rawDataStartIndexWithReserved() {
+ return rawData, fmt.Errorf("share %s is too short to contain raw data", s)
+ }
+
+ return s.data[s.rawDataStartIndexWithReserved():], nil
+}
+
+func (s *Share) rawDataStartIndexWithReserved() int {
+ isStart, err := s.IsSequenceStart()
+ if err != nil {
+ panic(err)
+ }
+
+ index := appconsts.NamespaceSize + appconsts.ShareInfoBytes
+ if isStart {
+ index += appconsts.SequenceLenBytes
+ }
+
+ return index
+}
+
 // RawData returns the raw share data. The raw share data does not contain the
 // namespace ID, info byte, sequence length, or reserved bytes.
 func (s *Share) RawData() (rawData []byte, err error) {
@@ -171,6 +194,51 @@ func (s *Share) RawData() (rawData []byte, err error) {
  return s.data[s.rawDataStartIndex():], nil
 }
 
+// RawDataWithReserved returns the raw share data while taking reserved bytes into account.
+func (s *Share) RawDataUsingReserved() (rawData []byte, err error) {
+ rawDataStartIndexUsingReserved, err := s.rawDataStartIndexUsingReserved()
+ if err != nil {
+ return nil, err
+ }
+
+ // This means share is the last share and does not have any transaction beginning in it
+ if rawDataStartIndexUsingReserved == 0 {
+ return []byte{}, nil
+ }
+ if len(s.data) < rawDataStartIndexUsingReserved {
+ return rawData, fmt.Errorf("share %s is too short to contain raw data", s)
+ }
+
+ return s.data[rawDataStartIndexUsingReserved:], nil
+}
+
+// rawDataStartIndexUsingReserved returns the start index of raw data while accounting for
+// reserved bytes, if it exists in the share.
+func (s *Share) rawDataStartIndexUsingReserved() (int, error) {
+ isStart, err := s.IsSequenceStart()
+ if err != nil {
+ return 0, err
+ }
+ isCompact, err := s.IsCompactShare()
+ if err != nil {
+ return 0, err
+ }
+
+ index := appconsts.NamespaceSize + appconsts.ShareInfoBytes
+ if isStart {
+ index += appconsts.SequenceLenBytes
+ }
+
+ if isCompact {
+ reservedBytes, err := ParseReservedBytes(s.data[index : index+appconsts.CompactShareReservedBytes])
+ if err != nil {
+ return 0, err
+ }
+ return int(reservedBytes), nil
+ }
+ return index, nil
+}
+
 func (s *Share) rawDataStartIndex() int {
  isStart, err := s.IsSequenceStart()
  if err != nil {
@@ -180,17 +248,15 @@ func (s *Share) rawDataStartIndex() int {
  if err != nil {
  panic(err)
  }
- if isStart && isCompact {
- return appconsts.NamespaceSize + appconsts.ShareInfoBytes + appconsts.SequenceLenBytes + appconsts.CompactShareReservedBytes
- } else if isStart && !isCompact {
- return appconsts.NamespaceSize + appconsts.ShareInfoBytes + appconsts.SequenceLenBytes
- } else if !isStart && isCompact {
- return appconsts.NamespaceSize + appconsts.ShareInfoBytes + appconsts.CompactShareReservedBytes
- } else if !isStart && !isCompact {
- return appconsts.NamespaceSize + appconsts.ShareInfoBytes
- } else {
- panic(fmt.Sprintf("unable to determine the rawDataStartIndex for share %s", s.data))
+
+ index := appconsts.NamespaceSize + appconsts.ShareInfoBytes
+ if isStart {
+ index += appconsts.SequenceLenBytes
+ }
+ if isCompact {
+ index += appconsts.CompactShareReservedBytes
  }
+ return index
 }
 
 func ToBytes(shares []Share) (bytes [][]byte) {

third_party/celestia-app/shares/split_compact_shares.go

@@ -51,6 +51,25 @@ func NewCompactShareSplitter(ns appns.Namespace, shareVersion uint8) *CompactSha
  }
 }
 
+// NewCompactShareSplitterWithIsCompactFalse returns a CompactShareSplitter using the provided
+// namespace and shareVersion. Also, sets isCompact in the builder to false.
+func NewCompactShareSplitterWithIsCompactFalse(ns appns.Namespace, shareVersion uint8) *CompactShareSplitter {
+ builder := NewBuilder(ns, shareVersion, true)
+ builder.isCompactShare = false
+ sb, err := builder.Init()
+ if err != nil {
+ panic(err)
+ }
+
+ return &CompactShareSplitter{
+ shares: []Share{},
+ namespace: ns,
+ shareVersion: shareVersion,
+ shareRanges: map[coretypes.TxKey]ShareRange{},
+ shareBuilder: sb,
+ }
+}
+
 // WriteTx adds the delimited data for the provided tx to the underlying compact
 // share splitter.
 func (css *CompactShareSplitter) WriteTx(tx coretypes.Tx) error {
@@ -73,6 +92,36 @@ func (css *CompactShareSplitter) WriteTx(tx coretypes.Tx) error {
  return nil
 }
 
+// write adds the delimited data to the underlying compact shares.
+func (css *CompactShareSplitter) WriteWithNoReservedBytes(rawData []byte) error {
+ if css.done {
+ // remove the last element
+ if !css.shareBuilder.IsEmptyShare() {
+ css.shares = css.shares[:len(css.shares)-1]
+ }
+ css.done = false
+ }
+
+ for {
+ rawDataLeftOver := css.shareBuilder.AddData(rawData)
+ if rawDataLeftOver == nil {
+ break
+ }
+ if err := css.stackPendingWithIsCompactFalse(); err != nil {
+ return err
+ }
+
+ rawData = rawDataLeftOver
+ }
+
+ if css.shareBuilder.AvailableBytes() == 0 {
+ if err := css.stackPendingWithIsCompactFalse(); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
 // write adds the delimited data to the underlying compact shares.
 func (css *CompactShareSplitter) write(rawData []byte) error {
  if css.done {
@@ -120,6 +169,21 @@ func (css *CompactShareSplitter) stackPending() error {
  return err
 }
 
+// stackPending will build & add the pending share to accumulated shares
+func (css *CompactShareSplitter) stackPendingWithIsCompactFalse() error {
+ pendingShare, err := css.shareBuilder.Build()
+ if err != nil {
+ return err
+ }
+ css.shares = append(css.shares, *pendingShare)
+
+ // Now we need to create a new builder
+ builder := NewBuilder(css.namespace, css.shareVersion, false)
+ builder.isCompactShare = false
+ css.shareBuilder, err = builder.Init()
+ return err
+}
+
 // Export finalizes and returns the underlying compact shares and a map of
 // shareRanges. All share ranges in the map of shareRanges will be offset (i.e.
 // incremented) by the shareRangeOffset provided. shareRangeOffset should be 0