Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

optimize in-memory transaction store #1063

Merged
merged 8 commits into from
Nov 6, 2023
Merged

optimize in-memory transaction store #1063

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5f6c60e
optimize in-memory transaction store
tsachiherman Nov 2, 2023
acac65b
fix test
tsachiherman Nov 2, 2023
fb563df
add memory consumption test
tsachiherman Nov 2, 2023
30fc756
Merge branch 'main' into tsachi/optimize-transaction-store
tsachiherman Nov 2, 2023
5c2a4cf
update per code review.
tsachiherman Nov 3, 2023
b7741b4
Merge branch 'tsachi/optimize-transaction-store' of https://github.co…
tsachiherman Nov 3, 2023
7d0b0a0
update:
tsachiherman Nov 3, 2023
8d5588f
Merge branch 'main' into tsachi/optimize-transaction-store
2opremio Nov 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion cmd/soroban-rpc/internal/ingest/service_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ func TestIngestion(t *testing.T) {
Type: xdr.ScAddressTypeScAddressTypeContract,
ContractId: &contractID,
}
xdrTrue := true
operationChanges := xdr.LedgerEntryChanges{
{
Type: xdr.LedgerEntryChangeTypeLedgerEntryState,
Expand All @@ -125,6 +126,10 @@ func TestIngestion(t *testing.T) {
Sym: &persistentKey,
},
Durability: xdr.ContractDataDurabilityPersistent,
Val: xdr.ScVal{
Type: xdr.ScValTypeScvBool,
B: &xdrTrue,
},
},
},
},
Expand All @@ -145,6 +150,10 @@ func TestIngestion(t *testing.T) {
Sym: &persistentKey,
},
Durability: xdr.ContractDataDurabilityPersistent,
Val: xdr.ScVal{
Type: xdr.ScValTypeScvBool,
B: &xdrTrue,
},
},
},
},
Expand All @@ -161,6 +170,10 @@ func TestIngestion(t *testing.T) {
Sym: &persistentKey,
},
Durability: xdr.ContractDataDurabilityTemporary,
Val: xdr.ScVal{
Type: xdr.ScValTypeScvBool,
B: &xdrTrue,
},
},
},
}
Expand Down Expand Up @@ -202,7 +215,14 @@ func TestIngestion(t *testing.T) {
},
TxProcessing: []xdr.TransactionResultMeta{
{
Result: xdr.TransactionResultPair{TransactionHash: firstTxHash},
Result: xdr.TransactionResultPair{
TransactionHash: firstTxHash,
Result: xdr.TransactionResult{
Result: xdr.TransactionResultResult{
Results: &[]xdr.OperationResult{},
},
},
},
FeeProcessing: xdr.LedgerEntryChanges{},
TxApplyProcessing: xdr.TransactionMeta{
V: 3,
Expand Down
25 changes: 6 additions & 19 deletions cmd/soroban-rpc/internal/methods/get_transaction.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package methods

import (
"context"
"encoding/base64"
"encoding/hex"
"fmt"

Expand Down Expand Up @@ -99,25 +100,11 @@ func GetTransaction(getter transactionGetter, request GetTransactionRequest) (Ge
response.FeeBump = tx.FeeBump
response.Ledger = int64(tx.Ledger.Sequence)
response.LedgerCloseTime = tx.Ledger.CloseTime
if response.ResultXdr, err = xdr.MarshalBase64(tx.Result); err != nil {
return GetTransactionResponse{}, &jrpc2.Error{
Code: jrpc2.InternalError,
Message: err.Error(),
}
}
if response.EnvelopeXdr, err = xdr.MarshalBase64(tx.Envelope); err != nil {
return GetTransactionResponse{}, &jrpc2.Error{
Code: jrpc2.InternalError,
Message: err.Error(),
}
}
if response.ResultMetaXdr, err = xdr.MarshalBase64(tx.Meta); err != nil {
return GetTransactionResponse{}, &jrpc2.Error{
Code: jrpc2.InternalError,
Message: err.Error(),
}
}
if tx.Result.Successful() {

response.ResultXdr = base64.StdEncoding.EncodeToString(tx.Result)
response.EnvelopeXdr = base64.StdEncoding.EncodeToString(tx.Envelope)
response.ResultMetaXdr = base64.StdEncoding.EncodeToString(tx.Meta)
if tx.Successful {
response.Status = TransactionStatusSuccess
} else {
response.Status = TransactionStatusFailed
Expand Down
40 changes: 20 additions & 20 deletions cmd/soroban-rpc/internal/methods/get_transaction_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"testing"

"github.com/stellar/go/xdr"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/stellar/go/network"

Expand Down Expand Up @@ -115,33 +115,33 @@ func txEnvelope(acctSeq uint32) xdr.TransactionEnvelope {
func TestGetTransaction(t *testing.T) {
store := transactions.NewMemoryStore(interfaces.MakeNoOpDeamon(), "passphrase", 100)
_, err := GetTransaction(store, GetTransactionRequest{"ab"})
assert.EqualError(t, err, "[-32602] unexpected hash length (2)")
require.EqualError(t, err, "[-32602] unexpected hash length (2)")
_, err = GetTransaction(store, GetTransactionRequest{"foo "})
assert.EqualError(t, err, "[-32602] incorrect hash: encoding/hex: invalid byte: U+006F 'o'")
require.EqualError(t, err, "[-32602] incorrect hash: encoding/hex: invalid byte: U+006F 'o'")

hash := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
tx, err := GetTransaction(store, GetTransactionRequest{hash})
assert.NoError(t, err)
assert.Equal(t, GetTransactionResponse{
require.NoError(t, err)
require.Equal(t, GetTransactionResponse{
Status: TransactionStatusNotFound,
}, tx)

meta := txMeta(1, true)
err = store.IngestTransactions(meta)
assert.NoError(t, err)
require.NoError(t, err)

xdrHash := txHash(1)
hash = hex.EncodeToString(xdrHash[:])
tx, err = GetTransaction(store, GetTransactionRequest{hash})
assert.NoError(t, err)
require.NoError(t, err)

expectedTxResult, err := xdr.MarshalBase64(meta.V1.TxProcessing[0].Result.Result)
assert.NoError(t, err)
require.NoError(t, err)
expectedEnvelope, err := xdr.MarshalBase64(txEnvelope(1))
assert.NoError(t, err)
require.NoError(t, err)
expectedTxMeta, err := xdr.MarshalBase64(meta.V1.TxProcessing[0].TxApplyProcessing)
assert.NoError(t, err)
assert.Equal(t, GetTransactionResponse{
require.NoError(t, err)
require.Equal(t, GetTransactionResponse{
Status: TransactionStatusSuccess,
LatestLedger: 101,
LatestLedgerCloseTime: 2625,
Expand All @@ -159,12 +159,12 @@ func TestGetTransaction(t *testing.T) {
// ingest another (failed) transaction
meta = txMeta(2, false)
err = store.IngestTransactions(meta)
assert.NoError(t, err)
require.NoError(t, err)

// the first transaction should still be there
tx, err = GetTransaction(store, GetTransactionRequest{hash})
assert.NoError(t, err)
assert.Equal(t, GetTransactionResponse{
require.NoError(t, err)
require.Equal(t, GetTransactionResponse{
Status: TransactionStatusSuccess,
LatestLedger: 102,
LatestLedgerCloseTime: 2650,
Expand All @@ -184,16 +184,16 @@ func TestGetTransaction(t *testing.T) {
hash = hex.EncodeToString(xdrHash[:])

expectedTxResult, err = xdr.MarshalBase64(meta.V1.TxProcessing[0].Result.Result)
assert.NoError(t, err)
require.NoError(t, err)
expectedEnvelope, err = xdr.MarshalBase64(txEnvelope(2))
assert.NoError(t, err)
require.NoError(t, err)
expectedTxMeta, err = xdr.MarshalBase64(meta.V1.TxProcessing[0].TxApplyProcessing)
assert.NoError(t, err)
require.NoError(t, err)

tx, err = GetTransaction(store, GetTransactionRequest{hash})
assert.NoError(t, err)
assert.NoError(t, err)
assert.Equal(t, GetTransactionResponse{
require.NoError(t, err)
require.NoError(t, err)
require.Equal(t, GetTransactionResponse{
Status: TransactionStatusFailed,
LatestLedger: 102,
LatestLedgerCloseTime: 2650,
Expand Down
28 changes: 19 additions & 9 deletions cmd/soroban-rpc/internal/transactions/transactions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,11 @@ import (

type transaction struct {
bucket *ledgerbucketwindow.LedgerBucket[[]xdr.Hash]
result xdr.TransactionResult
meta xdr.TransactionMeta
envelope xdr.TransactionEnvelope
result []byte // encoded XDR of xdr.TransactionResult
meta []byte // encoded XDR of xdr.TransactionMeta
envelope []byte // encoded XDR of xdr.TransactionEnvelope
feeBump bool
successful bool
applicationOrder int32
}

Expand Down Expand Up @@ -92,11 +93,18 @@ func (m *MemoryStore) IngestTransactions(ledgerCloseMeta xdr.LedgerCloseMeta) er
}
transactions[i] = transaction{
bucket: &bucket,
result: tx.Result.Result,
meta: tx.UnsafeMeta,
envelope: tx.Envelope,
feeBump: tx.Envelope.IsFeeBump(),
applicationOrder: int32(tx.Index),
successful: tx.Result.Result.Successful(),
}
if transactions[i].result, err = tx.Result.Result.MarshalBinary(); err != nil {
return err
}
if transactions[i].meta, err = tx.UnsafeMeta.MarshalBinary(); err != nil {
return err
}
if transactions[i].envelope, err = tx.Envelope.MarshalBinary(); err != nil {
return err
}
if transactions[i].feeBump {
innerHash := tx.Result.InnerHash()
Expand Down Expand Up @@ -135,11 +143,12 @@ type LedgerInfo struct {
}

type Transaction struct {
Result xdr.TransactionResult
Meta xdr.TransactionMeta
Envelope xdr.TransactionEnvelope
Result []byte // XDR encoded xdr.TransactionResult
Meta []byte // XDR encoded xdr.TransactionMeta
Envelope []byte // XDR encoded xdr.TransactionEnvelope
FeeBump bool
ApplicationOrder int32
Successful bool
Ledger LedgerInfo
}

Expand Down Expand Up @@ -191,6 +200,7 @@ func (m *MemoryStore) GetTransaction(hash xdr.Hash) (Transaction, bool, StoreRan
Meta: internalTx.meta,
Envelope: internalTx.envelope,
FeeBump: internalTx.feeBump,
Successful: internalTx.successful,
ApplicationOrder: internalTx.applicationOrder,
Ledger: LedgerInfo{
Sequence: internalTx.bucket.LedgerSeq,
Expand Down
Loading
Loading