prototype

blob/service.go

@@ -51,7 +51,8 @@ func NewService(
 	}
 }
 
-// SubmitOptions contains the information about fee and gasLimit price in order to configure the Submit request.
+// SubmitOptions contains the information about fee and gasLimit price in order to configure the
+// Submit request.
 type SubmitOptions struct {
 	Fee      int64
 	GasLimit uint64

share/eds/file.go

@@ -0,0 +1,174 @@
+package eds
+
+import (
+	"io"
+	"os"
+
+	"golang.org/x/exp/mmap"
+
+	"github.com/celestiaorg/celestia-app/pkg/wrapper"
+	"github.com/celestiaorg/nmt"
+	"github.com/celestiaorg/rsmt2d"
+
+	"github.com/celestiaorg/celestia-node/share"
+)
+
+// File
+// * immutable
+// * versionable
+// TODO:
+//   - Cache Rows and Cols
+//   - Avoid storing constant shares, like padding
+type File struct {
+	path string
+	h    Header
+	f    fileBackend
+}
+
+type fileBackend interface {
+	io.ReaderAt
+	io.Closer
+}
+
+func OpenFile(path string) (*File, error) {
+	f, err := mmap.Open(path)
+	if err != nil {
+		return nil, err
+	}
+
+	h, err := ReadHeaderAt(f, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	return &File{
+		path: path,
+		h:    h,
+		f:    f,
+	}, nil
+}
+
+// TODO: Allow setting features
+func CreateFile(path string, eds *rsmt2d.ExtendedDataSquare) (*File, error) {
+	f, err := os.Create(path)
+	if err != nil {
+		return nil, err
+	}
+
+	h := Header{
+		ShareSize:  uint16(len(eds.GetCell(0, 0))), // TODO: rsmt2d should expose this field
+		SquareSize: uint32(eds.Width()),
+	}
+
+	if _, err = h.WriteTo(f); err != nil {
+		return nil, err
+	}
+
+	for _, shr := range eds.Flattened() {
+		// TOOD: Buffer and write as single?
+		if _, err := f.Write(shr); err != nil {
+			return nil, err
+		}
+	}
+
+	return &File{
+		path: path,
+		f:    f,
+		h:    h,
+	}, err
+}
+
+func (f *File) Close() error {
+	return f.f.Close()
+}
+
+func (f *File) Header() Header {
+	return f.h
+}
+
+func (f *File) Axis(idx int, tp rsmt2d.Axis) ([]share.Share, error) {
+	// TODO: Add Col support
+	shrLn := int64(f.h.ShareSize)
+	sqrLn := int64(f.h.SquareSize)
+	rwwLn := shrLn * sqrLn
+
+	offset := int64(idx)*rwwLn + HeaderSize
+	rowdata := make([]byte, rwwLn)
+	if _, err := f.f.ReadAt(rowdata, offset); err != nil {
+		return nil, err
+	}
+
+	row := make([]share.Share, sqrLn)
+	for i := range row {
+		row[i] = rowdata[int64(i)*shrLn : (int64(i)+1)*shrLn]
+	}
+	return row, nil
+}
+
+func (f *File) Share(idx int) (share.Share, error) {
+	// TODO: Check the cache first
+	shrLn := int64(f.h.ShareSize)
+
+	offset := int64(idx)*shrLn + HeaderSize
+	shr := make(share.Share, shrLn)
+	if _, err := f.f.ReadAt(shr, offset); err != nil {
+		return nil, err
+	}
+	return shr, nil
+}
+
+func (f *File) ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, *nmt.Proof, error) {
+	// TODO: Cache the axis as well as computed tree
+	sqrLn := int(f.h.SquareSize)
+	rowIdx := idx / sqrLn
+	shrs, err := f.Axis(rowIdx, axis)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(f.h.SquareSize/2), uint(rowIdx))
+	for _, shr := range shrs {
+		err = tree.Push(shr)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
+	shrIdx := idx % sqrLn
+	proof, err := tree.ProveRange(shrIdx, shrIdx+1)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return shrs[shrIdx], &proof, nil
+}
+
+func (f *File) EDS() (*rsmt2d.ExtendedDataSquare, error) {
+	shrLn := int(f.h.ShareSize)
+	sqrLn := int(f.h.SquareSize)
+
+	buf := make([]byte, sqrLn*sqrLn*shrLn)
+	if _, err := f.f.ReadAt(buf, HeaderSize); err != nil {
+		return nil, err
+	}
+
+	shrs := make([][]byte, sqrLn*sqrLn)
+	for i := 0; i < sqrLn; i++ {
+		for j := 0; j < sqrLn; j++ {
+			coord := i*sqrLn + j
+			shrs[coord] = buf[coord*shrLn : (coord+1)*shrLn]
+		}
+	}
+
+	treeFn := func(_ rsmt2d.Axis, index uint) rsmt2d.Tree {
+		tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(f.h.SquareSize/2), index)
+		return &tree
+	}
+
+	eds, err := rsmt2d.ImportExtendedDataSquare(shrs, share.DefaultRSMT2DCodec(), treeFn)
+	if err != nil {
+		return nil, err
+	}
+
+	return eds, nil
+}

share/eds/file_header.go

@@ -0,0 +1,61 @@
+package eds
+
+import (
+	"encoding/binary"
+	"io"
+)
+
+const HeaderSize = 32
+
+type Header struct {
+	// User set features
+	// TODO: Add codec
+	Version     uint8
+	Compression uint8
+	Extensions  map[string]string
+	// Taken directly from EDS
+	ShareSize  uint16
+	SquareSize uint32
+}
+
+func (h *Header) WriteTo(w io.Writer) (int64, error) {
+	buf := make([]byte, HeaderSize)
+	buf[0] = h.Version
+	buf[1] = h.Compression
+	binary.LittleEndian.PutUint16(buf[2:4], h.ShareSize)
+	binary.LittleEndian.PutUint32(buf[4:12], h.SquareSize)
+	// TODO: Extensions
+	n, err := w.Write(buf)
+	return int64(n), err
+}
+
+func (h *Header) ReadFrom(r io.Reader) (int64, error) {
+	buf := make([]byte, HeaderSize)
+	n, err := io.ReadFull(r, buf)
+	if err != nil {
+		return int64(n), err
+	}
+
+	h.Version = buf[0]
+	h.Compression = buf[1]
+	h.ShareSize = binary.LittleEndian.Uint16(buf[2:4])
+	h.SquareSize = binary.LittleEndian.Uint32(buf[4:12])
+
+	// TODO: Extensions
+	return int64(n), err
+}
+
+func ReadHeaderAt(r io.ReaderAt, offset int64) (Header, error) {
+	h := Header{}
+	buf := make([]byte, HeaderSize)
+	_, err := r.ReadAt(buf, offset)
+	if err != nil {
+		return h, err
+	}
+
+	h.Version = buf[0]
+	h.Compression = buf[1]
+	h.ShareSize = binary.LittleEndian.Uint16(buf[2:4])
+	h.SquareSize = binary.LittleEndian.Uint32(buf[4:12])
+	return h, nil
+}

share/eds/file_store.go

@@ -0,0 +1,4 @@
+package eds
+
+type FileStore struct {
+}

share/eds/file_test.go

@@ -0,0 +1,58 @@
+package eds
+
+import (
+	"crypto/sha256"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/celestiaorg/rsmt2d"
+
+	"github.com/celestiaorg/celestia-node/share"
+	"github.com/celestiaorg/celestia-node/share/eds/edstest"
+)
+
+func TestFile(t *testing.T) {
+	path := t.TempDir() + "/testfile"
+	eds := edstest.RandEDS(t, 16)
+
+	fl, err := CreateFile(path, eds)
+	require.NoError(t, err)
+	err = fl.Close()
+	require.NoError(t, err)
+
+	fl, err = OpenFile(path)
+	require.NoError(t, err)
+
+	for i := 0; i < int(eds.Width()); i++ {
+		row, err := fl.Axis(i, rsmt2d.Row)
+		require.NoError(t, err)
+		assert.EqualValues(t, eds.Row(uint(i)), row)
+	}
+
+	width := int(eds.Width())
+	for i := 0; i < width*2; i++ {
+		row, col := uint(i/width), uint(i%width)
+		shr, err := fl.Share(i)
+		require.NoError(t, err)
+		assert.EqualValues(t, eds.GetCell(row, col), shr)
+
+		shr, proof, err := fl.ShareWithProof(i, rsmt2d.Row)
+		require.NoError(t, err)
+		assert.EqualValues(t, eds.GetCell(row, col), shr)
+
+		roots, err := eds.RowRoots()
+		require.NoError(t, err)
+
+		ok := proof.VerifyInclusion(sha256.New(), share.GetNamespace(shr).ToNMT(), [][]byte{shr}, roots[row])
+		assert.True(t, ok)
+	}
+
+	out, err := fl.EDS()
+	require.NoError(t, err)
+	assert.True(t, eds.Equals(out))
+
+	err = fl.Close()
+	require.NoError(t, err)
+}

share/ipld/blockserv.go

@@ -26,5 +26,5 @@ type allowlist struct{}
 
 func (a allowlist) IsAllowed(code uint64) bool {
 	// we allow all codes except home-baked sha256NamespaceFlagged
-	return code == sha256NamespaceFlagged
+	return code == sha256NamespaceFlagged || code == 0x7801
 }