diff --git a/go.mod b/go.mod index b1de82654b6..d892d558dc1 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,7 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca github.com/uber/jaeger-client-go v2.24.0+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible // indirect + github.com/vmihailenco/msgpack/v5 v5.3.4 github.com/wealdtech/go-ens/v3 v3.4.4 gitlab.com/nolash/go-mockbytes v0.0.7 go.opencensus.io v0.22.5 // indirect diff --git a/go.sum b/go.sum index f875d3f44c1..c640c1452cb 100644 --- a/go.sum +++ b/go.sum @@ -941,6 +941,10 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= +github.com/vmihailenco/msgpack/v5 v5.3.4 h1:qMKAwOV+meBw2Y8k9cVwAy7qErtYCwBzZ2ellBfvnqc= +github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/wealdtech/go-ens/v3 v3.4.4 h1:rgfjBqKj7L9ipVJOo/9XTQTKMcVERvxJpQBNUcIOnhs= github.com/wealdtech/go-ens/v3 v3.4.4/go.mod h1:X1ORiTz78XpHIhDATM1yZR9jxBPnV83mdX5Ty53IRb8= github.com/wealdtech/go-multicodec v1.2.0 h1:9AHSxcSE9F9r6ZvQLAO0EXCdM08QfYohaXmW3k6sSh4= diff --git a/openapi/SwarmCommon.yaml b/openapi/SwarmCommon.yaml index 393d4d4c414..eece755f6fc 100644 --- a/openapi/SwarmCommon.yaml +++ b/openapi/SwarmCommon.yaml @@ -321,6 +321,18 @@ components: type: integer usable: type: boolean + label: + type: string + depth: + type: integer + amount: + $ref: "#/components/schemas/BigInt" + bucketDepth: + type: integer + blockNumber: + type: integer + immutableFlag: + type: boolean Settlement: type: object diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index 94652855a78..506a762e884 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -11,6 +11,7 @@ import ( "errors" "io" "io/ioutil" + "math/big" "net/http" "net/http/httptest" "net/url" @@ -267,7 +268,7 @@ func TestPostageHeaderError(t *testing.T) { mockStorer = mock.NewStorer() mockStatestore = statestore.NewStateStore() logger = logging.New(ioutil.Discard, 5) - mp = mockpost.New(mockpost.WithIssuer(postage.NewStampIssuer("", "", batchOk, 11, 10, 1000))) + mp = mockpost.New(mockpost.WithIssuer(postage.NewStampIssuer("", "", batchOk, big.NewInt(3), 11, 10, 1000, true))) client, _, _ = newTestServer(t, testServerOptions{ Storer: mockStorer, Tags: tags.NewTags(mockStatestore, logger), diff --git a/pkg/api/feed_test.go b/pkg/api/feed_test.go index 11b8700499b..3e625a645f0 100644 --- a/pkg/api/feed_test.go +++ b/pkg/api/feed_test.go @@ -12,6 +12,7 @@ import ( "errors" "fmt" "io/ioutil" + "math/big" "net/http" "testing" @@ -154,7 +155,7 @@ func TestFeed_Post(t *testing.T) { logger = logging.New(ioutil.Discard, 0) tag = tags.NewTags(mockStatestore, logger) topic = "aabbcc" - mp = mockpost.New(mockpost.WithIssuer(postage.NewStampIssuer("", "", batchOk, 11, 10, 1000))) + mp = mockpost.New(mockpost.WithIssuer(postage.NewStampIssuer("", "", batchOk, big.NewInt(3), 11, 10, 1000, true))) mockStorer = mock.NewStorer() client, _, _ = newTestServer(t, testServerOptions{ Storer: mockStorer, diff --git a/pkg/api/postage.go b/pkg/api/postage.go index 88aa5a890c2..36ca9cb6fd1 100644 --- a/pkg/api/postage.go +++ b/pkg/api/postage.go @@ -12,6 +12,7 @@ import ( "net/http" "strconv" + "github.com/ethersphere/bee/pkg/bigint" "github.com/ethersphere/bee/pkg/jsonhttp" "github.com/ethersphere/bee/pkg/postage/postagecontract" "github.com/ethersphere/bee/pkg/sctx" @@ -97,25 +98,35 @@ func (s *server) postageCreateHandler(w http.ResponseWriter, r *http.Request) { } type postageStampResponse struct { - BatchID batchID `json:"batchID"` - Utilization uint32 `json:"utilization"` - Usable bool `json:"usable"` + BatchID batchID `json:"batchID"` + Utilization uint32 `json:"utilization"` + Usable bool `json:"usable"` + Label string `json:"label"` + Depth uint8 `json:"depth"` + Amount *bigint.BigInt `json:"amount"` + BucketDepth uint8 `json:"bucketDepth"` + BlockNumber uint64 `json:"blockNumber"` + ImmutableFlag bool `json:"immutableFlag"` } type postageStampsResponse struct { Stamps []postageStampResponse `json:"stamps"` } -func (s *server) postageGetStampsHandler(w http.ResponseWriter, r *http.Request) { - issuers := s.post.StampIssuers() +func (s *server) postageGetStampsHandler(w http.ResponseWriter, _ *http.Request) { resp := postageStampsResponse{} - for _, v := range issuers { - issuer := postageStampResponse{ - BatchID: v.ID(), - Utilization: v.Utilization(), - Usable: s.post.IssuerUsable(v), - } - resp.Stamps = append(resp.Stamps, issuer) + for _, v := range s.post.StampIssuers() { + resp.Stamps = append(resp.Stamps, postageStampResponse{ + BatchID: v.ID(), + Utilization: v.Utilization(), + Usable: s.post.IssuerUsable(v), + Label: v.Label(), + Depth: v.Depth(), + Amount: bigint.Wrap(v.Amount()), + BucketDepth: v.BucketDepth(), + BlockNumber: v.BlockNumber(), + ImmutableFlag: v.ImmutableFlag(), + }) } jsonhttp.OK(w, resp) } @@ -143,9 +154,15 @@ func (s *server) postageGetStampHandler(w http.ResponseWriter, r *http.Request) return } resp := postageStampResponse{ - BatchID: id, - Utilization: issuer.Utilization(), - Usable: s.post.IssuerUsable(issuer), + BatchID: id, + Utilization: issuer.Utilization(), + Usable: s.post.IssuerUsable(issuer), + Label: issuer.Label(), + Depth: issuer.Depth(), + Amount: bigint.Wrap(issuer.Amount()), + BucketDepth: issuer.BucketDepth(), + BlockNumber: issuer.BlockNumber(), + ImmutableFlag: issuer.ImmutableFlag(), } jsonhttp.OK(w, &resp) } diff --git a/pkg/api/postage_test.go b/pkg/api/postage_test.go index 7f7ae4fc2f1..a52dc59b682 100644 --- a/pkg/api/postage_test.go +++ b/pkg/api/postage_test.go @@ -14,6 +14,7 @@ import ( "testing" "github.com/ethersphere/bee/pkg/api" + "github.com/ethersphere/bee/pkg/bigint" "github.com/ethersphere/bee/pkg/jsonhttp" "github.com/ethersphere/bee/pkg/jsonhttp/jsonhttptest" "github.com/ethersphere/bee/pkg/postage" @@ -192,16 +193,23 @@ func TestPostageCreateStamp(t *testing.T) { } func TestPostageGetStamps(t *testing.T) { - mp := mockpost.New(mockpost.WithIssuer(postage.NewStampIssuer("", "", batchOk, 11, 10, 1000))) + si := postage.NewStampIssuer("", "", batchOk, big.NewInt(3), 11, 10, 1000, true) + mp := mockpost.New(mockpost.WithIssuer(si)) client, _, _ := newTestServer(t, testServerOptions{Post: mp}) jsonhttptest.Request(t, client, http.MethodGet, "/stamps", http.StatusOK, jsonhttptest.WithExpectedJSONResponse(&api.PostageStampsResponse{ Stamps: []api.PostageStampResponse{ { - BatchID: batchOk, - Utilization: 0, - Usable: true, + BatchID: batchOk, + Utilization: si.Utilization(), + Usable: true, + Label: si.Label(), + Depth: si.Depth(), + Amount: bigint.Wrap(si.Amount()), + BucketDepth: si.BucketDepth(), + BlockNumber: si.BlockNumber(), + ImmutableFlag: si.ImmutableFlag(), }, }, }), @@ -209,15 +217,22 @@ func TestPostageGetStamps(t *testing.T) { } func TestPostageGetStamp(t *testing.T) { - mp := mockpost.New(mockpost.WithIssuer(postage.NewStampIssuer("", "", batchOk, 11, 10, 1000))) + si := postage.NewStampIssuer("", "", batchOk, big.NewInt(3), 11, 10, 1000, true) + mp := mockpost.New(mockpost.WithIssuer(si)) client, _, _ := newTestServer(t, testServerOptions{Post: mp}) t.Run("ok", func(t *testing.T) { jsonhttptest.Request(t, client, http.MethodGet, "/stamps/"+batchOkStr, http.StatusOK, jsonhttptest.WithExpectedJSONResponse(&api.PostageStampResponse{ - BatchID: batchOk, - Utilization: 0, - Usable: true, + BatchID: batchOk, + Utilization: si.Utilization(), + Usable: true, + Label: si.Label(), + Depth: si.Depth(), + Amount: bigint.Wrap(si.Amount()), + BucketDepth: si.BucketDepth(), + BlockNumber: si.BlockNumber(), + ImmutableFlag: si.ImmutableFlag(), }), ) }) diff --git a/pkg/api/pss_test.go b/pkg/api/pss_test.go index ea6c5368753..ecfa8a8dd97 100644 --- a/pkg/api/pss_test.go +++ b/pkg/api/pss_test.go @@ -11,6 +11,7 @@ import ( "encoding/hex" "fmt" "io/ioutil" + "math/big" "net/http" "net/url" "sync" @@ -185,7 +186,7 @@ func TestPssSend(t *testing.T) { mtx.Unlock() return err } - mp = mockpost.New(mockpost.WithIssuer(postage.NewStampIssuer("", "", batchOk, 11, 10, 1000))) + mp = mockpost.New(mockpost.WithIssuer(postage.NewStampIssuer("", "", batchOk, big.NewInt(3), 11, 10, 1000, true))) p = newMockPss(sendFn) client, _, _ = newTestServer(t, testServerOptions{ Pss: p, diff --git a/pkg/api/soc_test.go b/pkg/api/soc_test.go index 8e559642e2f..70615ad36aa 100644 --- a/pkg/api/soc_test.go +++ b/pkg/api/soc_test.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "fmt" "io/ioutil" + "math/big" "net/http" "testing" @@ -32,7 +33,7 @@ func TestSOC(t *testing.T) { mockStatestore = statestore.NewStateStore() logger = logging.New(ioutil.Discard, 0) tag = tags.NewTags(mockStatestore, logger) - mp = mockpost.New(mockpost.WithIssuer(postage.NewStampIssuer("", "", batchOk, 11, 10, 1000))) + mp = mockpost.New(mockpost.WithIssuer(postage.NewStampIssuer("", "", batchOk, big.NewInt(3), 11, 10, 1000, true))) mockStorer = mock.NewStorer() client, _, _ = newTestServer(t, testServerOptions{ Storer: mockStorer, diff --git a/pkg/postage/batch.go b/pkg/postage/batch.go index d0ec8efb920..a916542b9c8 100644 --- a/pkg/postage/batch.go +++ b/pkg/postage/batch.go @@ -23,7 +23,7 @@ type Batch struct { // MarshalBinary implements BinaryMarshaller. It will attempt to serialize the // postage batch to a byte slice. -// serialised as ID(32)|big endian value(32)|start block(8)|owner addr(20)|bucketDepth(1)|depth(1)|immutable(1) +// serialised as ID(32)|big endian value(32)|start block(8)|owner addr(20)|BucketDepth(1)|depth(1)|immutable(1) func (b *Batch) MarshalBinary() ([]byte, error) { out := make([]byte, 95) copy(out, b.ID) diff --git a/pkg/postage/mock/service.go b/pkg/postage/mock/service.go index 584958751a0..ed0030aad44 100644 --- a/pkg/postage/mock/service.go +++ b/pkg/postage/mock/service.go @@ -6,6 +6,7 @@ package mock import ( "errors" + "math/big" "github.com/ethersphere/bee/pkg/postage" ) @@ -54,7 +55,7 @@ func (m *mockPostage) StampIssuers() []*postage.StampIssuer { func (m *mockPostage) GetStampIssuer(id []byte) (*postage.StampIssuer, error) { if m.acceptAll { - return postage.NewStampIssuer("test fallback", "test identity", id, 24, 6, 1000), nil + return postage.NewStampIssuer("test fallback", "test identity", id, big.NewInt(3), 24, 6, 1000, true), nil } if m.i != nil { diff --git a/pkg/postage/postagecontract/contract.go b/pkg/postage/postagecontract/contract.go index b8d07bc98c2..6e87336080c 100644 --- a/pkg/postage/postagecontract/contract.go +++ b/pkg/postage/postagecontract/contract.go @@ -190,9 +190,11 @@ func (c *postageContract) CreateBatch(ctx context.Context, initialBalance *big.I label, c.owner.Hex(), batchID, - depth, + initialBalance, + createdEvent.Depth, createdEvent.BucketDepth, ev.BlockNumber, + createdEvent.ImmutableFlag, )) return createdEvent.BatchId[:], nil diff --git a/pkg/postage/service.go b/pkg/postage/service.go index 18c01f1d7ac..e7e9e241139 100644 --- a/pkg/postage/service.go +++ b/pkg/postage/service.go @@ -94,7 +94,7 @@ func (ps *service) IssuerUsable(st *StampIssuer) bool { // the batch creation, before we start using a stamp issuer. The threshold // is meant to allow enough time for upstream peers to see the batch and // hence validate the stamps issued - if cs.Block < st.blockNumber || (cs.Block-st.blockNumber) < blockThreshold { + if cs.Block < st.data.BlockNumber || (cs.Block-st.data.BlockNumber) < blockThreshold { return false } return true @@ -105,7 +105,7 @@ func (ps *service) GetStampIssuer(batchID []byte) (*StampIssuer, error) { ps.lock.Lock() defer ps.lock.Unlock() for _, st := range ps.issuers { - if bytes.Equal(batchID, st.batchID) { + if bytes.Equal(batchID, st.data.BatchID) { if !ps.IssuerUsable(st) { return nil, ErrNotUsable } diff --git a/pkg/postage/service_test.go b/pkg/postage/service_test.go index fa11bbaa607..9e300bbcf56 100644 --- a/pkg/postage/service_test.go +++ b/pkg/postage/service_test.go @@ -7,6 +7,7 @@ package postage_test import ( crand "crypto/rand" "io" + "math/big" "reflect" "testing" @@ -75,11 +76,12 @@ func TestGetStampIssuer(t *testing.T) { if i == 0 { continue } - if i < 4 { - ps.Add(postage.NewStampIssuer(string(id), "", id, 16, 8, validBlockNumber)) - } else { - ps.Add(postage.NewStampIssuer(string(id), "", id, 16, 8, validBlockNumber+uint64(i))) + + var shift uint64 = 0 + if i > 3 { + shift = uint64(i) } + ps.Add(postage.NewStampIssuer(string(id), "", id, big.NewInt(3), 16, 8, validBlockNumber+shift, true)) } t.Run("found", func(t *testing.T) { for _, id := range ids[1:4] { diff --git a/pkg/postage/stamp_test.go b/pkg/postage/stamp_test.go index 20e17eca986..5be6c3cc99a 100644 --- a/pkg/postage/stamp_test.go +++ b/pkg/postage/stamp_test.go @@ -6,6 +6,7 @@ package postage_test import ( "bytes" + "math/big" "testing" "github.com/ethersphere/bee/pkg/crypto" @@ -74,7 +75,7 @@ func TestValidStamp(t *testing.T) { b := postagetesting.MustNewBatch(postagetesting.WithOwner(owner)) bs := mock.New(mock.WithBatch(b)) signer := crypto.NewDefaultSigner(privKey) - issuer := postage.NewStampIssuer("label", "keyID", b.ID, b.Depth, b.BucketDepth, 1000) + issuer := postage.NewStampIssuer("label", "keyID", b.ID, big.NewInt(3), b.Depth, b.BucketDepth, 1000, true) stamper := postage.NewStamper(issuer, signer) // this creates a chunk with a mocked stamp. ValidStamp will override this diff --git a/pkg/postage/stamper.go b/pkg/postage/stamper.go index 83e758259ef..85a46b1fd66 100644 --- a/pkg/postage/stamper.go +++ b/pkg/postage/stamper.go @@ -43,7 +43,7 @@ func (st *stamper) Stamp(addr swarm.Address) (*Stamp, error) { return nil, err } ts := timestamp() - toSign, err := toSignDigest(addr.Bytes(), st.issuer.batchID, index, ts) + toSign, err := toSignDigest(addr.Bytes(), st.issuer.data.BatchID, index, ts) if err != nil { return nil, err } @@ -51,7 +51,7 @@ func (st *stamper) Stamp(addr swarm.Address) (*Stamp, error) { if err != nil { return nil, err } - return NewStamp(st.issuer.batchID, index, ts, sig), nil + return NewStamp(st.issuer.data.BatchID, index, ts, sig), nil } func timestamp() []byte { diff --git a/pkg/postage/stamper_test.go b/pkg/postage/stamper_test.go index dd85a69193a..a94e9531b30 100644 --- a/pkg/postage/stamper_test.go +++ b/pkg/postage/stamper_test.go @@ -8,6 +8,7 @@ import ( crand "crypto/rand" "errors" "io" + "math/big" "testing" "github.com/ethersphere/bee/pkg/crypto" @@ -90,8 +91,7 @@ func TestStamperStamping(t *testing.T) { // tests that Stamps returns with postage.ErrBucketFull iff // issuer has the corresponding collision bucket filled] t.Run("bucket full", func(t *testing.T) { - st := newTestStampIssuer(t, 1000) - st = postage.NewStampIssuer("", "", st.ID(), 12, 8, 1000) + st := postage.NewStampIssuer("", "", newTestStampIssuer(t, 1000).ID(), big.NewInt(3), 12, 8, 1000, true) stamper := postage.NewStamper(st, signer) // issue 1 stamp chunkAddr, _ := createStamp(t, stamper) diff --git a/pkg/postage/stampissuer.go b/pkg/postage/stampissuer.go index a7d6ed20026..44c6a178f07 100644 --- a/pkg/postage/stampissuer.go +++ b/pkg/postage/stampissuer.go @@ -6,55 +6,71 @@ package postage import ( "encoding/binary" + "math/big" "sync" "github.com/ethersphere/bee/pkg/swarm" + "github.com/vmihailenco/msgpack/v5" ) +// stampIssuerData groups related StampIssuer data. +// The data are factored out in order to make +// serialization/deserialization easier and at the same +// time not to export the fields outside of the package. +type stampIssuerData struct { + Label string `msgpack:"label"` // Label to identify the batch period/importance. + KeyID string `msgpack:"keyID"` // Owner identity. + BatchID []byte `msgpack:"batchID"` // The batch stamps are issued from. + BatchAmount *big.Int `msgpack:"batchAmount"` // Amount paid for the batch. + BatchDepth uint8 `msgpack:"batchDepth"` // Batch depth: batch size = 2^{depth}. + BucketDepth uint8 `msgpack:"bucketDepth"` // Bucket depth: the depth of collision Buckets uniformity. + Buckets []uint32 `msgpack:"buckets"` // Collision Buckets: counts per neighbourhoods (limited to 2^{batchdepth-bucketdepth}). + MaxBucketCount uint32 `msgpack:"maxBucketCount"` // the count of the fullest bucket + BlockNumber uint64 `msgpack:"blockNumber"` // BlockNumber when this batch was created + ImmutableFlag bool `msgpack:"immutableFlag"` // Specifies immutability of the created batch. +} + // StampIssuer is a local extension of a batch issuing stamps for uploads. // A StampIssuer instance extends a batch with bucket collision tracking // embedded in multiple Stampers, can be used concurrently. type StampIssuer struct { - label string // Label to identify the batch period/importance. - keyID string // Owner identity. - batchID []byte // The batch stamps are issued from. - batchDepth uint8 // Batch depth: batch size = 2^{depth}. - bucketDepth uint8 // Bucket depth: the depth of collision buckets uniformity. - mu sync.Mutex // Mutex for buckets. - buckets []uint32 // Collision buckets: counts per neighbourhoods (limited to 2^{batchdepth-bucketdepth}). - maxBucketCount uint32 // the count of the fullest bucket - blockNumber uint64 // blockNumber when this batch was created + bucketMu sync.Mutex + data stampIssuerData } // NewStampIssuer constructs a StampIssuer as an extension of a batch for local // upload. // -// bucketDepth must always be smaller than batchDepth otherwise inc() panics. -func NewStampIssuer(label, keyID string, batchID []byte, batchDepth, bucketDepth uint8, blockNumber uint64) *StampIssuer { +// BucketDepth must always be smaller than batchDepth otherwise inc() panics. +func NewStampIssuer(label, keyID string, batchID []byte, batchAmount *big.Int, batchDepth, bucketDepth uint8, blockNumber uint64, immutableFlag bool) *StampIssuer { return &StampIssuer{ - label: label, - keyID: keyID, - batchID: batchID, - batchDepth: batchDepth, - bucketDepth: bucketDepth, - buckets: make([]uint32, 1< st.maxBucketCount { - st.maxBucketCount = st.buckets[b] + si.data.Buckets[b]++ + if si.data.Buckets[b] > si.data.MaxBucketCount { + si.data.MaxBucketCount = si.data.Buckets[b] } return indexToBytes(b, bucketCount), nil } @@ -84,71 +100,57 @@ func bytesToIndex(buf []byte) (bucket, index uint32) { } // Label returns the label of the issuer. -func (st *StampIssuer) Label() string { - return st.label -} - -// MarshalBinary gives the byte slice serialisation of a StampIssuer: -// = label[32]|keyID[32]|batchID[32]|batchDepth[1]|bucketDepth[1]|blockNumber[8]|size_0[4]|size_1[4]|.... -func (st *StampIssuer) MarshalBinary() ([]byte, error) { - buf := make([]byte, 32+32+32+1+1+8+4*(1<