diff --git a/cmd/soroban-rpc/internal/methods/simulate_transaction.go b/cmd/soroban-rpc/internal/methods/simulate_transaction.go index 374737ddd..b0042b880 100644 --- a/cmd/soroban-rpc/internal/methods/simulate_transaction.go +++ b/cmd/soroban-rpc/internal/methods/simulate_transaction.go @@ -28,13 +28,19 @@ type SimulateHostFunctionResult struct { XDR string `json:"xdr"` } +type RestorePreamble struct { + TransactionData string `json:"transactionData"` // SorobanTransactionData XDR in base64 + MinResourceFee int64 `json:"minResourceFee,string"` +} + type SimulateTransactionResponse struct { Error string `json:"error,omitempty"` TransactionData string `json:"transactionData"` // SorobanTransactionData XDR in base64 - Events []string `json:"events"` // DiagnosticEvent XDR in base64 MinResourceFee int64 `json:"minResourceFee,string"` - Results []SimulateHostFunctionResult `json:"results,omitempty"` // an array of the individual host function call results - Cost SimulateTransactionCost `json:"cost"` // the effective cpu and memory cost of the invoked transaction execution. + Events []string `json:"events,omitempty"` // DiagnosticEvent XDR in base64 + Results []SimulateHostFunctionResult `json:"results,omitempty"` // an array of the individual host function call results + Cost SimulateTransactionCost `json:"cost"` // the effective cpu and memory cost of the invoked transaction execution. + RestorePreamble RestorePreamble `json:"restorePreamble,omitempty"` // If present, it indicates that a prior RestoreFootprint is required LatestLedger int64 `json:"latestLedger,string"` } @@ -114,13 +120,23 @@ func NewSimulateTransactionHandler(logger *log.Entry, ledgerEntryReader db.Ledge } } + var results []SimulateHostFunctionResult + if result.Result != "" { + results = append(results, SimulateHostFunctionResult{ + XDR: result.Result, + Auth: result.Auth, + }) + } + restorePreable := RestorePreamble{} + if result.PreRestoreTransactionData != "" { + restorePreable = RestorePreamble{ + TransactionData: result.PreRestoreTransactionData, + MinResourceFee: result.PreRestoreMinFee, + } + } + return SimulateTransactionResponse{ - Results: []SimulateHostFunctionResult{ - { - XDR: result.Result, - Auth: result.Auth, - }, - }, + Results: results, Events: result.Events, TransactionData: result.TransactionData, MinResourceFee: result.MinFee, @@ -128,7 +144,8 @@ func NewSimulateTransactionHandler(logger *log.Entry, ledgerEntryReader db.Ledge CPUInstructions: result.CPUInstructions, MemoryBytes: result.MemoryBytes, }, - LatestLedger: int64(latestLedger), + LatestLedger: int64(latestLedger), + RestorePreamble: restorePreable, } }) } diff --git a/cmd/soroban-rpc/internal/preflight/preflight.go b/cmd/soroban-rpc/internal/preflight/preflight.go index d00457002..f209f51bd 100644 --- a/cmd/soroban-rpc/internal/preflight/preflight.go +++ b/cmd/soroban-rpc/internal/preflight/preflight.go @@ -61,28 +61,6 @@ func SnapshotSourceGet(handle C.uintptr_t, cLedgerKey *C.char, includeExpired C. return C.CString(out) } -// SnapshotSourceHas takes LedgerKey XDR in base64 and returns whether it exists -// It's used by the Rust preflight code to obtain ledger entries. -// -//export SnapshotSourceHas -func SnapshotSourceHas(handle C.uintptr_t, cLedgerKey *C.char) C.int { - h := cgo.Handle(handle).Value().(snapshotSourceHandle) - ledgerKeyB64 := C.GoString(cLedgerKey) - var ledgerKey xdr.LedgerKey - if err := xdr.SafeUnmarshalBase64(ledgerKeyB64, &ledgerKey); err != nil { - panic(err) - } - present, _, err := h.readTx.GetLedgerEntry(ledgerKey, false) - if err != nil { - h.logger.WithError(err).Error("SnapshotSourceHas(): GetLedgerEntry() failed") - return 0 - } - if present { - return 1 - } - return 0 -} - //export FreeGoCString func FreeGoCString(str *C.char) { C.free(unsafe.Pointer(str)) @@ -99,13 +77,15 @@ type PreflightParameters struct { } type Preflight struct { - Events []string // DiagnosticEvents XDR in base64 - TransactionData string // SorobanTransactionData XDR in base64 - MinFee int64 - Result string // XDR SCVal in base64 - Auth []string // SorobanAuthorizationEntrys XDR in base64 - CPUInstructions uint64 - MemoryBytes uint64 + Events []string // DiagnosticEvents XDR in base64 + TransactionData string // SorobanTransactionData XDR in base64 + MinFee int64 + Result string // XDR SCVal in base64 + Auth []string // SorobanAuthorizationEntrys XDR in base64 + CPUInstructions uint64 + MemoryBytes uint64 + PreRestoreTransactionData string // SorobanTransactionData XDR in base64 + PreRestoreMinFee int64 } // GoNullTerminatedStringSlice transforms a C NULL-terminated char** array to a Go string slice @@ -240,13 +220,15 @@ func GoPreflight(result *C.CPreflightResult) (Preflight, error) { } preflight := Preflight{ - Events: GoNullTerminatedStringSlice(result.events), - TransactionData: C.GoString(result.transaction_data), - MinFee: int64(result.min_fee), - Result: C.GoString(result.result), - Auth: GoNullTerminatedStringSlice(result.auth), - CPUInstructions: uint64(result.cpu_instructions), - MemoryBytes: uint64(result.memory_bytes), + Events: GoNullTerminatedStringSlice(result.events), + TransactionData: C.GoString(result.transaction_data), + MinFee: int64(result.min_fee), + Result: C.GoString(result.result), + Auth: GoNullTerminatedStringSlice(result.auth), + CPUInstructions: uint64(result.cpu_instructions), + MemoryBytes: uint64(result.memory_bytes), + PreRestoreTransactionData: C.GoString(result.pre_restore_transaction_data), + PreRestoreMinFee: int64(result.pre_restore_min_fee), } return preflight, nil } diff --git a/cmd/soroban-rpc/internal/preflight/preflight_test.go b/cmd/soroban-rpc/internal/preflight/preflight_test.go index 44bfcee26..392cba9cd 100644 --- a/cmd/soroban-rpc/internal/preflight/preflight_test.go +++ b/cmd/soroban-rpc/internal/preflight/preflight_test.go @@ -305,6 +305,7 @@ func getPreflightParameters(t testing.TB, inMemory bool) PreflightParameters { } func TestGetPreflight(t *testing.T) { + params := getPreflightParameters(t, false) _, err := GetPreflight(context.Background(), params) require.NoError(t, err) diff --git a/cmd/soroban-rpc/internal/test/captive-core-integration-tests.cfg b/cmd/soroban-rpc/internal/test/captive-core-integration-tests.cfg index 3562052f8..01b41a2ec 100644 --- a/cmd/soroban-rpc/internal/test/captive-core-integration-tests.cfg +++ b/cmd/soroban-rpc/internal/test/captive-core-integration-tests.cfg @@ -7,7 +7,7 @@ FAILURE_SAFETY=0 ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true # Lower the expiration of persistent ledger entries # so that ledger expiration/restoring becomes testeable -TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=20 +TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=10 [[VALIDATORS]] NAME="local_core" diff --git a/cmd/soroban-rpc/internal/test/simulate_transaction_test.go b/cmd/soroban-rpc/internal/test/simulate_transaction_test.go index 646d26376..c8b0243b9 100644 --- a/cmd/soroban-rpc/internal/test/simulate_transaction_test.go +++ b/cmd/soroban-rpc/internal/test/simulate_transaction_test.go @@ -127,32 +127,33 @@ func getContractID(t *testing.T, sourceAccount string, salt [32]byte, networkPas return hashedContractID } -func preflightTransactionParams(t *testing.T, client *jrpc2.Client, params txnbuild.TransactionParams) txnbuild.TransactionParams { +func simulateTransactionFromTxParams(t *testing.T, client *jrpc2.Client, params txnbuild.TransactionParams) methods.SimulateTransactionResponse { savedAutoIncrement := params.IncrementSequenceNum params.IncrementSequenceNum = false tx, err := txnbuild.NewTransaction(params) - params.IncrementSequenceNum = savedAutoIncrement assert.NoError(t, err) - assert.Len(t, params.Operations, 1) + params.IncrementSequenceNum = savedAutoIncrement txB64, err := tx.Base64() assert.NoError(t, err) - request := methods.SimulateTransactionRequest{Transaction: txB64} var response methods.SimulateTransactionResponse err = client.CallResult(context.Background(), "simulateTransaction", request, &response) assert.NoError(t, err) + return response +} + +func preflightTransactionParamsLocally(t *testing.T, params txnbuild.TransactionParams, response methods.SimulateTransactionResponse) txnbuild.TransactionParams { if !assert.Empty(t, response.Error) { fmt.Println(response.Error) } var transactionData xdr.SorobanTransactionData - err = xdr.SafeUnmarshalBase64(response.TransactionData, &transactionData) - + err := xdr.SafeUnmarshalBase64(response.TransactionData, &transactionData) require.NoError(t, err) - require.Len(t, response.Results, 1) op := params.Operations[0] switch v := op.(type) { case *txnbuild.InvokeHostFunction: + require.Len(t, response.Results, 1) v.Ext = xdr.TransactionExt{ V: 1, SorobanData: &transactionData, @@ -166,11 +167,13 @@ func preflightTransactionParams(t *testing.T, client *jrpc2.Client, params txnbu } v.Auth = auth case *txnbuild.BumpFootprintExpiration: + require.Len(t, response.Results, 0) v.Ext = xdr.TransactionExt{ V: 1, SorobanData: &transactionData, } case *txnbuild.RestoreFootprint: + require.Len(t, response.Results, 0) v.Ext = xdr.TransactionExt{ V: 1, SorobanData: &transactionData, @@ -185,6 +188,11 @@ func preflightTransactionParams(t *testing.T, client *jrpc2.Client, params txnbu return params } +func preflightTransactionParams(t *testing.T, client *jrpc2.Client, params txnbuild.TransactionParams) txnbuild.TransactionParams { + response := simulateTransactionFromTxParams(t, client, params) + return preflightTransactionParamsLocally(t, params, response) +} + func TestSimulateTransactionSucceeds(t *testing.T) { test := NewTest(t) @@ -192,7 +200,7 @@ func TestSimulateTransactionSucceeds(t *testing.T) { client := jrpc2.NewClient(ch, nil) sourceAccount := keypair.Root(StandaloneNetworkPassphrase).Address() - tx, err := txnbuild.NewTransaction(txnbuild.TransactionParams{ + params := txnbuild.TransactionParams{ SourceAccount: &txnbuild.SimpleAccount{ AccountID: sourceAccount, Sequence: 0, @@ -206,21 +214,14 @@ func TestSimulateTransactionSucceeds(t *testing.T) { Preconditions: txnbuild.Preconditions{ TimeBounds: txnbuild.NewInfiniteTimeout(), }, - }) - require.NoError(t, err) - txB64, err := tx.Base64() - require.NoError(t, err) - request := methods.SimulateTransactionRequest{Transaction: txB64} + } + result := simulateTransactionFromTxParams(t, client, params) testContractIdBytes := xdr.ScBytes(testContractId) expectedXdr := xdr.ScVal{ Type: xdr.ScValTypeScvBytes, Bytes: &testContractIdBytes, } - - var result methods.SimulateTransactionResponse - err = client.CallResult(context.Background(), "simulateTransaction", request, &result) - assert.NoError(t, err) assert.Greater(t, result.LatestLedger, int64(0)) assert.Greater(t, result.Cost.CPUInstructions, uint64(0)) assert.Greater(t, result.Cost.MemoryBytes, uint64(0)) @@ -248,7 +249,7 @@ func TestSimulateTransactionSucceeds(t *testing.T) { // First, decode and compare the transaction data so we get a decent diff if it fails. var transactionData xdr.SorobanTransactionData - err = xdr.SafeUnmarshalBase64(result.TransactionData, &transactionData) + err := xdr.SafeUnmarshalBase64(result.TransactionData, &transactionData) assert.NoError(t, err) assert.Equal(t, expectedTransactionData, transactionData) @@ -261,7 +262,7 @@ func TestSimulateTransactionSucceeds(t *testing.T) { // test operation which does not have a source account withoutSourceAccountOp := createInstallContractCodeOperation("", testContract) - tx, err = txnbuild.NewTransaction(txnbuild.TransactionParams{ + params = txnbuild.TransactionParams{ SourceAccount: &txnbuild.SimpleAccount{ AccountID: sourceAccount, Sequence: 0, @@ -273,20 +274,14 @@ func TestSimulateTransactionSucceeds(t *testing.T) { Preconditions: txnbuild.Preconditions{ TimeBounds: txnbuild.NewInfiniteTimeout(), }, - }) - require.NoError(t, err) - - txB64, err = tx.Base64() + } require.NoError(t, err) - request = methods.SimulateTransactionRequest{Transaction: txB64} - var resultForRequestWithoutOpSource methods.SimulateTransactionResponse - err = client.CallResult(context.Background(), "simulateTransaction", request, &resultForRequestWithoutOpSource) - assert.NoError(t, err) + resultForRequestWithoutOpSource := simulateTransactionFromTxParams(t, client, params) assert.Equal(t, result, resultForRequestWithoutOpSource) // test that operation source account takes precedence over tx source account - tx, err = txnbuild.NewTransaction(txnbuild.TransactionParams{ + params = txnbuild.TransactionParams{ SourceAccount: &txnbuild.SimpleAccount{ AccountID: keypair.Root("test passphrase").Address(), Sequence: 0, @@ -300,15 +295,9 @@ func TestSimulateTransactionSucceeds(t *testing.T) { Preconditions: txnbuild.Preconditions{ TimeBounds: txnbuild.NewInfiniteTimeout(), }, - }) - require.NoError(t, err) - txB64, err = tx.Base64() - require.NoError(t, err) - request = methods.SimulateTransactionRequest{Transaction: txB64} + } - var resultForRequestWithDifferentTxSource methods.SimulateTransactionResponse - err = client.CallResult(context.Background(), "simulateTransaction", request, &resultForRequestWithDifferentTxSource) - assert.NoError(t, err) + resultForRequestWithDifferentTxSource := simulateTransactionFromTxParams(t, client, params) assert.GreaterOrEqual(t, resultForRequestWithDifferentTxSource.LatestLedger, result.LatestLedger) // apart from latest ledger the response should be the same resultForRequestWithDifferentTxSource.LatestLedger = result.LatestLedger @@ -499,7 +488,7 @@ func TestSimulateInvokeContractTransactionSucceeds(t *testing.T) { require.Contains(t, metrics, "soroban_rpc_json_rpc_request_duration_seconds_count{endpoint=\"simulateTransaction\",status=\"ok\"} 3") require.Contains(t, metrics, "soroban_rpc_preflight_pool_request_ledger_get_duration_seconds_count{status=\"ok\",type=\"db\"} 3") require.Contains(t, metrics, "soroban_rpc_preflight_pool_request_ledger_get_duration_seconds_count{status=\"ok\",type=\"all\"} 3") - require.Contains(t, metrics, "soroban_rpc_preflight_pool_request_ledger_entries_fetched_sum 52") + require.Contains(t, metrics, "soroban_rpc_preflight_pool_request_ledger_entries_fetched_sum 55") } func TestSimulateTransactionError(t *testing.T) { @@ -521,7 +510,7 @@ func TestSimulateTransactionError(t *testing.T) { Args: nil, }, } - tx, err := txnbuild.NewTransaction(txnbuild.TransactionParams{ + params := txnbuild.TransactionParams{ SourceAccount: &txnbuild.SimpleAccount{ AccountID: keypair.Root(StandaloneNetworkPassphrase).Address(), Sequence: 0, @@ -533,15 +522,8 @@ func TestSimulateTransactionError(t *testing.T) { Preconditions: txnbuild.Preconditions{ TimeBounds: txnbuild.NewInfiniteTimeout(), }, - }) - require.NoError(t, err) - txB64, err := tx.Base64() - require.NoError(t, err) - request := methods.SimulateTransactionRequest{Transaction: txB64} - - var result methods.SimulateTransactionResponse - err = client.CallResult(context.Background(), "simulateTransaction", request, &result) - assert.NoError(t, err) + } + result := simulateTransactionFromTxParams(t, client, params) assert.Greater(t, result.LatestLedger, int64(0)) assert.Contains(t, result.Error, "MissingValue") } @@ -553,7 +535,7 @@ func TestSimulateTransactionMultipleOperations(t *testing.T) { client := jrpc2.NewClient(ch, nil) sourceAccount := keypair.Root(StandaloneNetworkPassphrase).Address() - tx, err := txnbuild.NewTransaction(txnbuild.TransactionParams{ + params := txnbuild.TransactionParams{ SourceAccount: &txnbuild.SimpleAccount{ AccountID: keypair.Root(StandaloneNetworkPassphrase).Address(), Sequence: 0, @@ -568,15 +550,9 @@ func TestSimulateTransactionMultipleOperations(t *testing.T) { Preconditions: txnbuild.Preconditions{ TimeBounds: txnbuild.NewInfiniteTimeout(), }, - }) - require.NoError(t, err) - txB64, err := tx.Base64() - require.NoError(t, err) - request := methods.SimulateTransactionRequest{Transaction: txB64} + } - var result methods.SimulateTransactionResponse - err = client.CallResult(context.Background(), "simulateTransaction", request, &result) - assert.NoError(t, err) + result := simulateTransactionFromTxParams(t, client, params) assert.Equal( t, methods.SimulateTransactionResponse{ @@ -592,7 +568,7 @@ func TestSimulateTransactionWithoutInvokeHostFunction(t *testing.T) { ch := jhttp.NewChannel(test.sorobanRPCURL(), nil) client := jrpc2.NewClient(ch, nil) - tx, err := txnbuild.NewTransaction(txnbuild.TransactionParams{ + params := txnbuild.TransactionParams{ SourceAccount: &txnbuild.SimpleAccount{ AccountID: keypair.Root(StandaloneNetworkPassphrase).Address(), Sequence: 0, @@ -606,15 +582,8 @@ func TestSimulateTransactionWithoutInvokeHostFunction(t *testing.T) { Preconditions: txnbuild.Preconditions{ TimeBounds: txnbuild.NewInfiniteTimeout(), }, - }) - require.NoError(t, err) - txB64, err := tx.Base64() - require.NoError(t, err) - request := methods.SimulateTransactionRequest{Transaction: txB64} - - var result methods.SimulateTransactionResponse - err = client.CallResult(context.Background(), "simulateTransaction", request, &result) - assert.NoError(t, err) + } + result := simulateTransactionFromTxParams(t, client, params) assert.Equal( t, methods.SimulateTransactionResponse{ @@ -684,7 +653,7 @@ func TestSimulateTransactionBumpAndRestoreFootprint(t *testing.T) { sendSuccessfulTransaction(t, client, sourceAccount, tx) contractID := getContractID(t, address, testSalt, StandaloneNetworkPassphrase) - params = preflightTransactionParams(t, client, txnbuild.TransactionParams{ + invokeIncPresistentEntryParams := txnbuild.TransactionParams{ SourceAccount: &account, IncrementSequenceNum: true, Operations: []txnbuild.Operation{ @@ -698,7 +667,8 @@ func TestSimulateTransactionBumpAndRestoreFootprint(t *testing.T) { Preconditions: txnbuild.Preconditions{ TimeBounds: txnbuild.NewInfiniteTimeout(), }, - }) + } + params = preflightTransactionParams(t, client, invokeIncPresistentEntryParams) tx, err = txnbuild.NewTransaction(params) assert.NoError(t, err) sendSuccessfulTransaction(t, client, sourceAccount, tx) @@ -726,11 +696,11 @@ func TestSimulateTransactionBumpAndRestoreFootprint(t *testing.T) { getLedgerEntryrequest := methods.GetLedgerEntryRequest{ Key: keyB64, } - var result methods.GetLedgerEntryResponse - err = client.CallResult(context.Background(), "getLedgerEntry", getLedgerEntryrequest, &result) + var getLedgerEntryResult methods.GetLedgerEntryResponse + err = client.CallResult(context.Background(), "getLedgerEntry", getLedgerEntryrequest, &getLedgerEntryResult) assert.NoError(t, err) var entry xdr.LedgerEntryData - assert.NoError(t, xdr.SafeUnmarshalBase64(result.XDR, &entry)) + assert.NoError(t, xdr.SafeUnmarshalBase64(getLedgerEntryResult.XDR, &entry)) initialExpirationSeq, ok := entry.ExpirationLedgerSeq() assert.True(t, ok) @@ -761,28 +731,31 @@ func TestSimulateTransactionBumpAndRestoreFootprint(t *testing.T) { assert.NoError(t, err) sendSuccessfulTransaction(t, client, sourceAccount, tx) - err = client.CallResult(context.Background(), "getLedgerEntry", getLedgerEntryrequest, &result) + err = client.CallResult(context.Background(), "getLedgerEntry", getLedgerEntryrequest, &getLedgerEntryResult) assert.NoError(t, err) - assert.NoError(t, xdr.SafeUnmarshalBase64(result.XDR, &entry)) + assert.NoError(t, xdr.SafeUnmarshalBase64(getLedgerEntryResult.XDR, &entry)) newExpirationSeq, ok := entry.ExpirationLedgerSeq() assert.True(t, ok) assert.Greater(t, newExpirationSeq, initialExpirationSeq) // Wait until it expires - expired := false - for i := 0; i < 50; i++ { - err = client.CallResult(context.Background(), "getLedgerEntry", getLedgerEntryrequest, &result) - if err != nil { - expired = true - t.Logf("ledger entry expired") - break + waitForExpiration := func() { + expired := false + for i := 0; i < 50; i++ { + err = client.CallResult(context.Background(), "getLedgerEntry", getLedgerEntryrequest, &getLedgerEntryResult) + if err != nil { + expired = true + t.Logf("ledger entry expired") + break + } + assert.NoError(t, xdr.SafeUnmarshalBase64(getLedgerEntryResult.XDR, &entry)) + t.Log("waiting for ledger entry to expire at ledger", entry.MustContractData().ExpirationLedgerSeq+1) + time.Sleep(time.Second) } - assert.NoError(t, xdr.SafeUnmarshalBase64(result.XDR, &entry)) - t.Log("waiting for ledger entry to expire at ledger", entry.MustContractData().ExpirationLedgerSeq+1) - time.Sleep(time.Second) + require.True(t, expired) } - require.True(t, expired) + waitForExpiration() // and restore it params = preflightTransactionParams(t, client, txnbuild.TransactionParams{ @@ -810,4 +783,37 @@ func TestSimulateTransactionBumpAndRestoreFootprint(t *testing.T) { tx, err = txnbuild.NewTransaction(params) assert.NoError(t, err) sendSuccessfulTransaction(t, client, sourceAccount, tx) + + // Wait for expiration again and check the pre-restore field when trying to exec the contract again + waitForExpiration() + + simulationResult := simulateTransactionFromTxParams(t, client, invokeIncPresistentEntryParams) + assert.NotZero(t, simulationResult.RestorePreamble) + + params = preflightTransactionParamsLocally(t, + txnbuild.TransactionParams{ + SourceAccount: &account, + IncrementSequenceNum: true, + Operations: []txnbuild.Operation{ + &txnbuild.RestoreFootprint{}, + }, + Preconditions: txnbuild.Preconditions{ + TimeBounds: txnbuild.NewInfiniteTimeout(), + }, + }, + methods.SimulateTransactionResponse{ + TransactionData: simulationResult.RestorePreamble.TransactionData, + MinResourceFee: simulationResult.RestorePreamble.MinResourceFee, + }, + ) + tx, err = txnbuild.NewTransaction(params) + assert.NoError(t, err) + sendSuccessfulTransaction(t, client, sourceAccount, tx) + + // Finally, we should be able to send the inc host function invocation now that we + // have pre-restored the entries + params = preflightTransactionParamsLocally(t, invokeIncPresistentEntryParams, simulationResult) + tx, err = txnbuild.NewTransaction(params) + assert.NoError(t, err) + sendSuccessfulTransaction(t, client, sourceAccount, tx) } diff --git a/cmd/soroban-rpc/internal/test/stellar-core-integration-tests.cfg b/cmd/soroban-rpc/internal/test/stellar-core-integration-tests.cfg index 48e6c589b..4b4d0787d 100644 --- a/cmd/soroban-rpc/internal/test/stellar-core-integration-tests.cfg +++ b/cmd/soroban-rpc/internal/test/stellar-core-integration-tests.cfg @@ -17,7 +17,7 @@ DATABASE="postgresql://user=postgres password=mysecretpassword host=core-postgre ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true # Lower the expiration of persistent ledger entries # so that ledger expiration/restoring becomes testeable -TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=20 +TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=10 [QUORUM_SET] THRESHOLD_PERCENT=100 diff --git a/cmd/soroban-rpc/lib/preflight.h b/cmd/soroban-rpc/lib/preflight.h index 176f77172..e620075ec 100644 --- a/cmd/soroban-rpc/lib/preflight.h +++ b/cmd/soroban-rpc/lib/preflight.h @@ -15,6 +15,7 @@ typedef struct CLedgerInfo { uint32_t auto_bump_ledgers; } CLedgerInfo; + typedef struct CPreflightResult { char *error; // Error string in case of error, otherwise null char **auth; // NULL terminated array of XDR SorobanAuthorizationEntrys in base64 @@ -24,6 +25,8 @@ typedef struct CPreflightResult { char **events; // NULL terminated array of XDR DiagnosticEvents in base64 uint64_t cpu_instructions; uint64_t memory_bytes; + char *pre_restore_transaction_data; // SorobanTransactionData XDR in base64 for a prerequired RestoreFootprint operation + int64_t pre_restore_min_fee; // Minimum recommended resource fee for a prerequired RestoreFootprint operation } CPreflightResult; CPreflightResult *preflight_invoke_hf_op(uintptr_t handle, // Go Handle to forward to SnapshotSourceGet and SnapshotSourceHas @@ -41,9 +44,6 @@ CPreflightResult *preflight_footprint_expiration_op(uintptr_t handle, // Go Hand // LedgerKey XDR in base64 string to LedgerEntry XDR in base64 string extern char *SnapshotSourceGet(uintptr_t handle, char *ledger_key, int include_expired); -// LedgerKey XDR in base64 string to bool -extern int SnapshotSourceHas(uintptr_t handle, char *ledger_key); - void free_preflight_result(CPreflightResult *result); extern void FreeGoCString(char *str); diff --git a/cmd/soroban-rpc/lib/preflight/src/fees.rs b/cmd/soroban-rpc/lib/preflight/src/fees.rs index 959c76eb5..66dd71a6d 100644 --- a/cmd/soroban-rpc/lib/preflight/src/fees.rs +++ b/cmd/soroban-rpc/lib/preflight/src/fees.rs @@ -10,13 +10,13 @@ use soroban_env_host::storage::{AccessType, Footprint, Storage}; use soroban_env_host::xdr; use soroban_env_host::xdr::ContractDataDurability::Persistent; use soroban_env_host::xdr::{ - BumpFootprintExpirationOp, ConfigSettingEntry, ConfigSettingId, ContractDataDurability, - DecoratedSignature, DiagnosticEvent, ExtensionPoint, InvokeHostFunctionOp, LedgerEntry, - LedgerEntryData, LedgerFootprint, LedgerKey, Memo, MuxedAccount, MuxedAccountMed25519, - Operation, OperationBody, Preconditions, RestoreFootprintOp, SequenceNumber, Signature, - SignatureHint, SorobanResources, SorobanTransactionData, Transaction, TransactionExt, - TransactionV1Envelope, Uint256, WriteXdr, + BumpFootprintExpirationOp, ConfigSettingEntry, ConfigSettingId, DecoratedSignature, + DiagnosticEvent, ExtensionPoint, InvokeHostFunctionOp, LedgerFootprint, LedgerKey, Memo, + MuxedAccount, MuxedAccountMed25519, Operation, OperationBody, Preconditions, + RestoreFootprintOp, SequenceNumber, Signature, SignatureHint, SorobanResources, + SorobanTransactionData, Transaction, TransactionExt, TransactionV1Envelope, Uint256, WriteXdr, }; +use state_expiration::{get_restored_ledger_sequence, ExpirableLedgerEntry}; use std::cmp::max; use std::convert::{TryFrom, TryInto}; use std::error; @@ -308,19 +308,18 @@ pub(crate) fn compute_bump_footprint_exp_transaction_data_and_min_fee( let mut rent_changes: Vec = Vec::new(); for key in (&footprint).read_only.as_vec() { let unmodified_entry = ledger_storage.get(key, false)?; - let (expiration_ledger, durability) = - get_bumpable_entry_expiration_and_durability(&unmodified_entry)?; + let size = (key.to_xdr()?.len() + unmodified_entry.to_xdr()?.len()) as u32; + let expirable_entry: Box = (&unmodified_entry).try_into()?; let new_expiration_ledger = current_ledger_seq + ledgers_to_expire; - if new_expiration_ledger <= expiration_ledger { - // The bump would be innefective + if new_expiration_ledger <= expirable_entry.expiration_ledger_seq() { + // The bump would be ineffective continue; } - let size = (key.to_xdr()?.len() + unmodified_entry.to_xdr()?.len()) as u32; let rent_change = LedgerEntryRentChange { - is_persistent: durability == Persistent, + is_persistent: expirable_entry.durability() == Persistent, old_size_bytes: size, new_size_bytes: size, - old_expiration_ledger: expiration_ledger, + old_expiration_ledger: expirable_entry.expiration_ledger_seq(), new_expiration_ledger, }; rent_changes.push(rent_change); @@ -362,24 +361,6 @@ pub(crate) fn compute_bump_footprint_exp_transaction_data_and_min_fee( ) } -fn get_bumpable_entry_expiration_and_durability( - ledger_entry: &LedgerEntry, -) -> Result<(u32, ContractDataDurability), Box> { - let res = match ledger_entry.data { - LedgerEntryData::ContractData(ref cd) => (cd.expiration_ledger_seq, cd.durability), - LedgerEntryData::ContractCode(ref cc) => (cc.expiration_ledger_seq, Persistent), - _ => { - let err = format!( - "Incorrect ledger entry type ({}) in footprint", - ledger_entry.data.name() - ) - .into(); - return Err(err); - } - }; - Ok(res) -} - pub(crate) fn compute_restore_footprint_transaction_data_and_min_fee( footprint: LedgerFootprint, ledger_storage: &ledger_storage::LedgerStorage, @@ -396,19 +377,20 @@ pub(crate) fn compute_restore_footprint_transaction_data_and_min_fee( let mut rent_changes: Vec = Vec::new(); for key in footprint.read_write.as_vec() { let unmodified_entry = ledger_storage.get(key, true)?; - let (expiration_ledger, durability) = - get_bumpable_entry_expiration_and_durability(&unmodified_entry)?; - if durability != Persistent { + let size = (key.to_xdr()?.len() + unmodified_entry.to_xdr()?.len()) as u32; + let expirable_entry: Box = (&unmodified_entry).try_into()?; + if expirable_entry.durability() != Persistent { let err = format!("Non-persistent key ({:?}) in footprint", key).into(); return Err(err); } - if current_ledger_seq <= expiration_ledger { + if !expirable_entry.has_expired(current_ledger_seq) { // noop (the entry hadn't expired) continue; } - let new_expiration_ledger = - current_ledger_seq + state_expiration.min_persistent_entry_expiration - 1; - let size = (key.to_xdr()?.len() + unmodified_entry.to_xdr()?.len()) as u32; + let new_expiration_ledger = get_restored_ledger_sequence( + current_ledger_seq, + state_expiration.min_persistent_entry_expiration, + ); let rent_change = LedgerEntryRentChange { is_persistent: true, old_size_bytes: 0, diff --git a/cmd/soroban-rpc/lib/preflight/src/ledger_storage.rs b/cmd/soroban-rpc/lib/preflight/src/ledger_storage.rs index 2b80985b0..3dac742b1 100644 --- a/cmd/soroban-rpc/lib/preflight/src/ledger_storage.rs +++ b/cmd/soroban-rpc/lib/preflight/src/ledger_storage.rs @@ -1,11 +1,15 @@ use base64::{engine::general_purpose::STANDARD as base64, DecodeError, Engine as _}; -use ledger_storage::Error::UnexpectedConfigLedgerEntryType; use soroban_env_host::storage::SnapshotSource; +use soroban_env_host::xdr::ContractDataDurability::Persistent; use soroban_env_host::xdr::{ ConfigSettingEntry, ConfigSettingId, Error as XdrError, LedgerEntry, LedgerEntryData, LedgerKey, LedgerKeyConfigSetting, ReadXdr, ScError, ScErrorCode, WriteXdr, }; use soroban_env_host::HostError; +use state_expiration::{restore_ledger_entry, ExpirableLedgerEntry}; +use std::cell::RefCell; +use std::collections::HashSet; +use std::convert::TryInto; use std::ffi::{CStr, CString, NulError}; use std::rc::Rc; use std::str::Utf8Error; @@ -20,9 +24,6 @@ extern "C" { ledger_key: *const libc::c_char, include_expired: libc::c_int, ) -> *const libc::c_char; - // TODO: this function is unnecessary, we can just look for null in SnapshotSourceGet - // LedgerKey XDR in base64 string to bool - fn SnapshotSourceHas(handle: libc::uintptr_t, ledger_key: *const libc::c_char) -> libc::c_int; } #[derive(thiserror::Error, Debug)] @@ -37,8 +38,8 @@ pub(crate) enum Error { DecodeError(#[from] DecodeError), #[error("utf8 error: {0}")] Utf8Error(#[from] Utf8Error), - #[error("unexpected ledger entry type for key {key_name}")] - UnexpectedConfigLedgerEntryType { key_name: String }, + #[error("unexpected config ledger entry for setting_id {setting_id}")] + UnexpectedConfigLedgerEntry { setting_id: String }, } impl Error { @@ -51,11 +52,85 @@ impl Error { } } +struct EntryRestoreTracker { + current_ledger_seq: u32, + min_persistent_entry_expiration: u32, + // RefCell is needed to mutate the hashset inside SnapshotSource::get(), which is an immutable method + ledger_keys_requiring_restore: RefCell>, +} + +impl EntryRestoreTracker { + pub(crate) fn track_and_restore(&self, key: &LedgerKey, entry: &mut LedgerEntry) { + if self.track(key, entry) { + restore_ledger_entry( + entry, + self.current_ledger_seq, + self.min_persistent_entry_expiration, + ); + } + } + + pub(crate) fn track(&self, key: &LedgerKey, entry: &LedgerEntry) -> bool { + let expirable_entry: Box = match entry.try_into() { + Ok(e) => e, + Err(_) => { + // Nothing to track, the entry isn't expirable + return false; + } + }; + if expirable_entry.durability() != Persistent + || !expirable_entry.has_expired(self.current_ledger_seq) + { + // Nothing to track, the entry isn't persistent (and thus not restorable) or + // it hasn't expired + return false; + } + self.ledger_keys_requiring_restore + .borrow_mut() + .insert(key.clone()); + return true; + } +} + pub(crate) struct LedgerStorage { - pub(crate) golang_handle: libc::uintptr_t, + golang_handle: libc::uintptr_t, + restore_tracker: Option, } impl LedgerStorage { + pub(crate) fn new(golang_handle: libc::uintptr_t) -> Self { + LedgerStorage { + golang_handle, + restore_tracker: None, + } + } + + pub(crate) fn with_restore_tracking( + golang_handle: libc::uintptr_t, + current_ledger_sequence: u32, + ) -> Result { + // First, we initialize it without the tracker, to get the minimum restore ledger from the network + let mut ledger_storage = LedgerStorage { + golang_handle, + restore_tracker: None, + }; + let setting_id = ConfigSettingId::StateExpiration; + let ConfigSettingEntry::StateExpiration(state_expiration) = + ledger_storage.get_configuration_setting(setting_id)? + else { + return Err( + Error::UnexpectedConfigLedgerEntry { setting_id: setting_id.name().to_string() } + ); + }; + // Now that we have the state expiration config, we can build the tracker + ledger_storage.restore_tracker = Some(EntryRestoreTracker { + current_ledger_seq: current_ledger_sequence, + ledger_keys_requiring_restore: RefCell::new(HashSet::new()), + min_persistent_entry_expiration: state_expiration.min_persistent_entry_expiration, + }); + return Ok(ledger_storage); + } + fn get_xdr_base64(&self, key: &LedgerKey, include_expired: bool) -> Result { let key_xdr = key.to_xdr_base64()?; let key_cstr = CString::new(key_xdr)?; @@ -79,18 +154,18 @@ impl LedgerStorage { Ok(str) } - pub fn get(&self, key: &LedgerKey, include_expired: bool) -> Result { + pub(crate) fn get(&self, key: &LedgerKey, include_expired: bool) -> Result { let base64_str = self.get_xdr_base64(key, include_expired)?; let entry = LedgerEntry::from_xdr_base64(base64_str)?; Ok(entry) } - pub fn get_xdr(&self, key: &LedgerKey, include_expired: bool) -> Result, Error> { + pub(crate) fn get_xdr(&self, key: &LedgerKey, include_expired: bool) -> Result, Error> { let base64_str = self.get_xdr_base64(key, include_expired)?; Ok(base64.decode(base64_str)?) } - pub fn get_configuration_setting( + pub(crate) fn get_configuration_setting( &self, setting_id: ConfigSettingId, ) -> Result { @@ -102,26 +177,42 @@ impl LedgerStorage { data: LedgerEntryData::ConfigSetting(cs), .. } => Ok(cs), - _ => Err(UnexpectedConfigLedgerEntryType { - key_name: setting_id.name().to_string(), + _ => Err(Error::UnexpectedConfigLedgerEntry { + setting_id: setting_id.name().to_string(), }), } } + + pub(crate) fn get_ledger_keys_requiring_restore(&self) -> HashSet { + match self.restore_tracker { + Some(ref t) => t.ledger_keys_requiring_restore.borrow().clone(), + None => HashSet::new(), + } + } } impl SnapshotSource for LedgerStorage { fn get(&self, key: &Rc) -> Result, HostError> { - let entry = self.get(key, false).map_err(|e| Error::to_host_error(&e))?; + let mut entry = ::get(self, key, self.restore_tracker.is_some()) + .map_err(|e| Error::to_host_error(&e))?; + if let Some(ref tracker) = self.restore_tracker { + // If the entry expired, we modify it to make it seem like it was restored + tracker.track_and_restore(key, &mut entry); + } Ok(entry.into()) } fn has(&self, key: &Rc) -> Result { - let key_xdr = key - .to_xdr_base64() - .map_err(|_| ScError::Value(ScErrorCode::InvalidInput))?; - let key_cstr = - CString::new(key_xdr).map_err(|_| ScError::Value(ScErrorCode::InvalidInput))?; - let res = unsafe { SnapshotSourceHas(self.golang_handle, key_cstr.as_ptr()) }; - Ok(res != 0) + let entry = match ::get(self, key, self.restore_tracker.is_some()) { + Err(e) => match e { + Error::NotFound => return Ok(false), + _ => return Err(Error::to_host_error(&e)), + }, + Ok(le) => le, + }; + if let Some(ref tracker) = self.restore_tracker { + tracker.track(key, &entry); + } + Ok(true) } } diff --git a/cmd/soroban-rpc/lib/preflight/src/lib.rs b/cmd/soroban-rpc/lib/preflight/src/lib.rs index 40c7b201d..425c0e44a 100644 --- a/cmd/soroban-rpc/lib/preflight/src/lib.rs +++ b/cmd/soroban-rpc/lib/preflight/src/lib.rs @@ -1,5 +1,6 @@ mod fees; mod ledger_storage; +mod state_expiration; extern crate base64; extern crate libc; @@ -14,13 +15,14 @@ use soroban_env_host::events::Events; use soroban_env_host::storage::Storage; use soroban_env_host::xdr::{ AccountId, ConfigSettingEntry, ConfigSettingId, DiagnosticEvent, InvokeHostFunctionOp, - LedgerFootprint, OperationBody, ReadXdr, ScVal, SorobanAddressCredentials, + LedgerFootprint, LedgerKey, OperationBody, ReadXdr, ScVal, SorobanAddressCredentials, SorobanAuthorizationEntry, SorobanCredentials, VecM, WriteXdr, }; use soroban_env_host::{DiagnosticLevel, Host, LedgerInfo}; -use std::convert::TryFrom; +use std::convert::{TryFrom, TryInto}; use std::error::Error; use std::ffi::{CStr, CString}; +use std::iter::FromIterator; use std::panic; use std::ptr::null_mut; use std::rc::Rc; @@ -67,6 +69,8 @@ pub struct CPreflightResult { pub events: *mut *mut libc::c_char, // NULL terminated array of XDR ContractEvents in base64 pub cpu_instructions: u64, pub memory_bytes: u64, + pub pre_restore_transaction_data: *mut libc::c_char, // SorobanTransactionData XDR in base64 for a prerequired RestoreFootprint operation + pub pre_restore_min_fee: i64, // Minimum recommended resource fee for a prerequired RestoreFootprint operation } fn preflight_error(str: String) -> *mut CPreflightResult { @@ -82,6 +86,8 @@ fn preflight_error(str: String) -> *mut CPreflightResult { events: null_mut(), cpu_instructions: 0, memory_bytes: 0, + pre_restore_transaction_data: null_mut(), + pre_restore_min_fee: 0, })) } @@ -115,12 +121,12 @@ fn preflight_invoke_hf_op_or_maybe_panic( let invoke_hf_op = InvokeHostFunctionOp::from_xdr_base64(invoke_hf_op_cstr.to_str()?)?; let source_account_cstr = unsafe { CStr::from_ptr(source_account) }; let source_account = AccountId::from_xdr_base64(source_account_cstr.to_str()?)?; - let storage = Storage::with_recording_footprint(Rc::new(LedgerStorage { - golang_handle: handle, - })); - let budget = get_budget_from_network_config_params(&LedgerStorage { - golang_handle: handle, - })?; + let ledger_storage = Rc::new(LedgerStorage::with_restore_tracking( + handle, + ledger_info.sequence_number, + )?); + let budget = get_budget_from_network_config_params(&ledger_storage)?; + let storage = Storage::with_recording_footprint(ledger_storage.clone()); let host = Host::with_storage_and_budget(storage, budget); // We make an assumption here: @@ -165,15 +171,34 @@ fn preflight_invoke_hf_op_or_maybe_panic( host_function: invoke_hf_op.host_function, auth: auths.clone(), }, - &LedgerStorage { - golang_handle: handle, - }, + &ledger_storage, &storage, &budget, &diagnostic_events, bucket_list_size, ledger_info.sequence_number, )?; + + let entries = ledger_storage.get_ledger_keys_requiring_restore(); + let (pre_restore_transaction_data, pre_restore_min_fee) = if entries.len() > 0 { + let read_write_vec: Vec = Vec::from_iter(entries); + let restore_footprint = LedgerFootprint { + read_only: VecM::default(), + read_write: read_write_vec.try_into()?, + }; + let (transaction_data, min_fee) = + fees::compute_restore_footprint_transaction_data_and_min_fee( + restore_footprint, + &ledger_storage, + bucket_list_size, + ledger_info.sequence_number, + )?; + let transaction_data_cstr = CString::new(transaction_data.to_xdr_base64()?)?; + (transaction_data_cstr.into_raw(), min_fee) + } else { + (null_mut(), 0) + }; + let transaction_data_cstr = CString::new(transaction_data.to_xdr_base64()?)?; Ok(CPreflightResult { error: null_mut(), @@ -184,6 +209,8 @@ fn preflight_invoke_hf_op_or_maybe_panic( events: diagnostic_events_to_c(diagnostic_events)?, cpu_instructions: budget.get_cpu_insns_consumed()?, memory_bytes: budget.get_mem_bytes_consumed()?, + pre_restore_transaction_data, + pre_restore_min_fee, }) } @@ -253,9 +280,7 @@ fn preflight_footprint_expiration_op_or_maybe_panic( let op_body = OperationBody::from_xdr_base64(op_body_cstr.to_str()?)?; let footprint_cstr = unsafe { CStr::from_ptr(footprint) }; let ledger_footprint = LedgerFootprint::from_xdr_base64(footprint_cstr.to_str()?)?; - let ledger_storage = &ledger_storage::LedgerStorage { - golang_handle: handle, - }; + let ledger_storage = &LedgerStorage::new(handle); match op_body { OperationBody::BumpFootprintExpiration(op) => preflight_bump_footprint_expiration( ledger_footprint, @@ -303,6 +328,8 @@ fn preflight_bump_footprint_expiration( events: null_mut(), cpu_instructions: 0, memory_bytes: 0, + pre_restore_transaction_data: null_mut(), + pre_restore_min_fee: 0, }) } @@ -328,6 +355,8 @@ fn preflight_restore_footprint( events: null_mut(), cpu_instructions: 0, memory_bytes: 0, + pre_restore_transaction_data: null_mut(), + pre_restore_min_fee: 0, }) } diff --git a/cmd/soroban-rpc/lib/preflight/src/state_expiration.rs b/cmd/soroban-rpc/lib/preflight/src/state_expiration.rs new file mode 100644 index 000000000..eed1dd31d --- /dev/null +++ b/cmd/soroban-rpc/lib/preflight/src/state_expiration.rs @@ -0,0 +1,69 @@ +use soroban_env_host::xdr::ContractDataDurability::Persistent; +use soroban_env_host::xdr::{ + ContractCodeEntry, ContractDataDurability, ContractDataEntry, LedgerEntry, LedgerEntryData, +}; +use std::convert::TryInto; + +pub(crate) trait ExpirableLedgerEntry { + fn durability(&self) -> ContractDataDurability; + fn expiration_ledger_seq(&self) -> u32; + fn has_expired(&self, current_ledger_seq: u32) -> bool { + current_ledger_seq > self.expiration_ledger_seq() + } +} + +impl ExpirableLedgerEntry for &ContractCodeEntry { + fn durability(&self) -> ContractDataDurability { + Persistent + } + + fn expiration_ledger_seq(&self) -> u32 { + self.expiration_ledger_seq + } +} + +impl ExpirableLedgerEntry for &ContractDataEntry { + fn durability(&self) -> ContractDataDurability { + self.durability + } + + fn expiration_ledger_seq(&self) -> u32 { + self.expiration_ledger_seq + } +} + +impl<'a> TryInto> for &'a LedgerEntry { + type Error = String; + + fn try_into(self) -> Result, Self::Error> { + match &self.data { + LedgerEntryData::ContractData(d) => Ok(Box::new(d)), + LedgerEntryData::ContractCode(c) => Ok(Box::new(c)), + _ => Err(format!( + "Incorrect ledger entry type ({}) in footprint", + self.data.name() + )), + } + } +} + +pub(crate) fn get_restored_ledger_sequence( + current_ledger_seq: u32, + min_persistent_entry_expiration: u32, +) -> u32 { + return current_ledger_seq + min_persistent_entry_expiration - 1; +} + +pub(crate) fn restore_ledger_entry( + ledger_entry: &mut LedgerEntry, + current_ledger_seq: u32, + min_persistent_entry_expiration: u32, +) { + let new_ledger_seq = + get_restored_ledger_sequence(current_ledger_seq, min_persistent_entry_expiration); + match &mut ledger_entry.data { + LedgerEntryData::ContractData(d) => d.expiration_ledger_seq = new_ledger_seq, + LedgerEntryData::ContractCode(c) => c.expiration_ledger_seq = new_ledger_seq, + _ => (), + } +}