From 6a0a621eda4115a0c8bb2c60e85c5b9bedced054 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Wed, 13 Dec 2023 10:32:26 -0400 Subject: [PATCH] Fix dependency-check script to support crate versions (#1129) --- cmd/soroban-rpc/lib/preflight/src/fees.rs | 2 +- .../lib/preflight/src/ledger_storage.rs | 116 +++++------------- cmd/soroban-rpc/lib/preflight/src/lib.rs | 101 ++++++++++++++- .../lib/preflight/src/state_ttl.rs | 6 +- scripts/check-dependencies.bash | 26 +++- 5 files changed, 153 insertions(+), 98 deletions(-) diff --git a/cmd/soroban-rpc/lib/preflight/src/fees.rs b/cmd/soroban-rpc/lib/preflight/src/fees.rs index 1cfc16b16e..25569eeedb 100644 --- a/cmd/soroban-rpc/lib/preflight/src/fees.rs +++ b/cmd/soroban-rpc/lib/preflight/src/fees.rs @@ -1,5 +1,5 @@ use anyhow::{bail, ensure, Context, Error, Result}; -use ledger_storage::LedgerStorage; +use ledger_storage::{LedgerGetter, LedgerStorage}; use soroban_env_host::budget::Budget; use soroban_env_host::e2e_invoke::{ extract_rent_changes, get_ledger_changes, LedgerEntryChange, TtlEntryMap, diff --git a/cmd/soroban-rpc/lib/preflight/src/ledger_storage.rs b/cmd/soroban-rpc/lib/preflight/src/ledger_storage.rs index 264355a140..5cb44a5b4c 100644 --- a/cmd/soroban-rpc/lib/preflight/src/ledger_storage.rs +++ b/cmd/soroban-rpc/lib/preflight/src/ledger_storage.rs @@ -1,28 +1,17 @@ -use sha2::Digest; use soroban_env_host::storage::SnapshotSource; use soroban_env_host::xdr::ContractDataDurability::{Persistent, Temporary}; use soroban_env_host::xdr::{ - ConfigSettingEntry, ConfigSettingId, Error as XdrError, Hash, LedgerEntry, LedgerEntryData, - LedgerKey, LedgerKeyConfigSetting, LedgerKeyTtl, Limits, ReadXdr, ScError, ScErrorCode, - TtlEntry, WriteXdr, + ConfigSettingEntry, ConfigSettingId, Error as XdrError, LedgerEntry, LedgerEntryData, + LedgerKey, LedgerKeyConfigSetting, Limits, ScError, ScErrorCode, WriteXdr, }; use soroban_env_host::HostError; -use state_ttl::{get_restored_ledger_sequence, is_live, TTLLedgerEntry}; +use state_ttl::{get_restored_ledger_sequence, TTLLedgerEntry}; use std::cell::RefCell; use std::collections::HashSet; use std::convert::TryInto; use std::ffi::NulError; use std::rc::Rc; use std::str::Utf8Error; -use {from_c_xdr, CXDR}; - -// Functions imported from Golang -extern "C" { - // Free Strings returned from Go functions - fn FreeGoXDR(xdr: CXDR); - // LedgerKey XDR in base64 string to LedgerEntry XDR in base64 string - fn SnapshotSourceGet(handle: libc::uintptr_t, ledger_key: CXDR) -> CXDR; -} #[derive(thiserror::Error, Debug)] pub(crate) enum Error { @@ -88,28 +77,36 @@ impl EntryRestoreTracker { } } +pub(crate) trait LedgerGetter { + fn get( + &self, + key: &LedgerKey, + include_not_live: bool, + ) -> anyhow::Result<(LedgerEntry, Option), Error>; +} + pub(crate) struct LedgerStorage { - golang_handle: libc::uintptr_t, + ledger_getter: Box, current_ledger_sequence: u32, restore_tracker: Option, } impl LedgerStorage { - pub(crate) fn new(golang_handle: libc::uintptr_t, current_ledger_sequence: u32) -> Self { + pub(crate) fn new(ledger_getter: Box, current_ledger_sequence: u32) -> Self { LedgerStorage { - golang_handle, + ledger_getter, current_ledger_sequence, restore_tracker: None, } } pub(crate) fn with_restore_tracking( - golang_handle: libc::uintptr_t, + ledger_getter: Box, current_ledger_sequence: u32, ) -> Result { // First, we initialize it without the tracker, to get the minimum restore ledger from the network let mut ledger_storage = LedgerStorage { - golang_handle, + ledger_getter, current_ledger_sequence, restore_tracker: None, }; @@ -129,66 +126,6 @@ impl LedgerStorage { Ok(ledger_storage) } - // Get the XDR, regardless of ttl - fn get_xdr_internal(&self, key_xdr: &mut Vec) -> Result, Error> { - let key_c_xdr = CXDR { - xdr: key_xdr.as_mut_ptr(), - len: key_xdr.len(), - }; - let res = unsafe { SnapshotSourceGet(self.golang_handle, key_c_xdr) }; - if res.xdr.is_null() { - return Err(Error::NotFound); - } - let v = from_c_xdr(res); - unsafe { FreeGoXDR(res) }; - Ok(v) - } - - pub(crate) fn get( - &self, - key: &LedgerKey, - include_not_live: bool, - ) -> Result<(LedgerEntry, Option), Error> { - let mut key_xdr = key.to_xdr(Limits::none())?; - let xdr = self.get_xdr_internal(&mut key_xdr)?; - - let live_until_ledger_seq = match key { - // TODO: it would probably be more efficient to do all of this in the Go side - // (e.g. it would allow us to query multiple entries at once) - LedgerKey::ContractData(_) | LedgerKey::ContractCode(_) => { - let key_hash: [u8; 32] = sha2::Sha256::digest(key_xdr).into(); - let ttl_key = LedgerKey::Ttl(LedgerKeyTtl { - key_hash: Hash(key_hash), - }); - let mut ttl_key_xdr = ttl_key.to_xdr(Limits::none())?; - let ttl_entry_xdr = self.get_xdr_internal(&mut ttl_key_xdr)?; - let ttl_entry = LedgerEntry::from_xdr(ttl_entry_xdr, Limits::none())?; - if let LedgerEntryData::Ttl(TtlEntry { - live_until_ledger_seq, - .. - }) = ttl_entry.data - { - Some(live_until_ledger_seq) - } else { - return Err(Error::UnexpectedLedgerEntryTypeForTtlKey { - ledger_entry_type: ttl_entry.data.name().to_string(), - }); - } - } - _ => None, - }; - - if !include_not_live - && live_until_ledger_seq.is_some() - && !is_live(live_until_ledger_seq.unwrap(), self.current_ledger_sequence) - { - return Err(Error::NotLive); - } - - let entry = LedgerEntry::from_xdr(xdr, Limits::none())?; - Ok((entry, live_until_ledger_seq)) - } - pub(crate) fn get_xdr( &self, key: &LedgerKey, @@ -196,7 +133,7 @@ impl LedgerStorage { ) -> Result, Error> { // TODO: this can be optimized since for entry types other than ContractCode/ContractData, // they don't need to be deserialized and serialized again - let (entry, _) = self.get(key, include_not_live)?; + let (entry, _) = self.ledger_getter.get(key, include_not_live)?; Ok(entry.to_xdr(Limits::none())?) } @@ -207,7 +144,7 @@ impl LedgerStorage { let key = LedgerKey::ConfigSetting(LedgerKeyConfigSetting { config_setting_id: setting_id, }); - match self.get(&key, false)? { + match self.ledger_getter.get(&key, false)? { ( LedgerEntry { data: LedgerEntryData::ConfigSetting(cs), @@ -229,10 +166,20 @@ impl LedgerStorage { } } +impl LedgerGetter for LedgerStorage { + fn get( + &self, + key: &LedgerKey, + include_not_live: bool, + ) -> anyhow::Result<(LedgerEntry, Option), Error> { + self.ledger_getter.get(key, include_not_live) + } +} + impl SnapshotSource for LedgerStorage { fn get(&self, key: &Rc) -> Result<(Rc, Option), HostError> { if let Some(ref tracker) = self.restore_tracker { - let mut entry_and_ttl = self.get(key, true)?; + let mut entry_and_ttl = self.ledger_getter.get(key, true)?; // Explicitly discard temporary ttl'ed entries if let Ok(ttl_entry) = TryInto::>::try_into(&entry_and_ttl) { if ttl_entry.durability() == Temporary @@ -246,7 +193,10 @@ impl SnapshotSource for LedgerStorage { tracker.track_and_restore(self.current_ledger_sequence, key, &entry_and_ttl); return Ok((entry_and_ttl.0.into(), entry_and_ttl.1)); } - let entry_and_ttl = ::get(self, key, false).map_err(HostError::from)?; + let entry_and_ttl = self + .ledger_getter + .get(key, false) + .map_err(HostError::from)?; Ok((entry_and_ttl.0.into(), entry_and_ttl.1)) } diff --git a/cmd/soroban-rpc/lib/preflight/src/lib.rs b/cmd/soroban-rpc/lib/preflight/src/lib.rs index f49a131e87..bc2523fc79 100644 --- a/cmd/soroban-rpc/lib/preflight/src/lib.rs +++ b/cmd/soroban-rpc/lib/preflight/src/lib.rs @@ -10,11 +10,12 @@ extern crate sha2; extern crate soroban_env_host; use anyhow::{Context, Result}; -use ledger_storage::LedgerStorage; +use ledger_storage::{Error, LedgerGetter, LedgerStorage}; use preflight::PreflightResult; use sha2::{Digest, Sha256}; use soroban_env_host::xdr::{ - AccountId, InvokeHostFunctionOp, LedgerFootprint, Limits, OperationBody, ReadXdr, WriteXdr, + AccountId, Hash, InvokeHostFunctionOp, LedgerEntry, LedgerEntryData, LedgerFootprint, + LedgerKey, LedgerKeyTtl, Limits, OperationBody, ReadXdr, TtlEntry, WriteXdr, }; use soroban_env_host::LedgerInfo; use std::ffi::{CStr, CString}; @@ -158,8 +159,13 @@ fn preflight_invoke_hf_op_or_maybe_panic( let invoke_hf_op = InvokeHostFunctionOp::from_xdr(from_c_xdr(invoke_hf_op), Limits::none()).unwrap(); let source_account = AccountId::from_xdr(from_c_xdr(source_account), Limits::none()).unwrap(); - let ledger_storage = LedgerStorage::with_restore_tracking(handle, ledger_info.sequence_number) - .context("cannot create LedgerStorage")?; + let go_storage = GoLedgerStorage { + golang_handle: handle, + current_ledger_sequence: ledger_info.sequence_number, + }; + let ledger_storage = + LedgerStorage::with_restore_tracking(Box::new(go_storage), ledger_info.sequence_number) + .context("cannot create LedgerStorage")?; let result = preflight::preflight_invoke_hf_op( ledger_storage, bucket_list_size, @@ -199,7 +205,11 @@ fn preflight_footprint_ttl_op_or_maybe_panic( ) -> Result { let op_body = OperationBody::from_xdr(from_c_xdr(op_body), Limits::none()).unwrap(); let footprint = LedgerFootprint::from_xdr(from_c_xdr(footprint), Limits::none()).unwrap(); - let ledger_storage = &LedgerStorage::new(handle, current_ledger_seq); + let go_storage = GoLedgerStorage { + golang_handle: handle, + current_ledger_sequence: current_ledger_seq, + }; + let ledger_storage = &LedgerStorage::new(Box::new(go_storage), current_ledger_seq); let result = preflight::preflight_footprint_ttl_op( ledger_storage, bucket_list_size, @@ -342,3 +352,84 @@ fn from_c_xdr(xdr: CXDR) -> Vec { let s = unsafe { slice::from_raw_parts(xdr.xdr, xdr.len) }; s.to_vec() } + +// Functions imported from Golang +extern "C" { + // Free Strings returned from Go functions + fn FreeGoXDR(xdr: CXDR); + // LedgerKey XDR in base64 string to LedgerEntry XDR in base64 string + fn SnapshotSourceGet(handle: libc::uintptr_t, ledger_key: CXDR) -> CXDR; +} + +struct GoLedgerStorage { + golang_handle: libc::uintptr_t, + current_ledger_sequence: u32, +} + +impl GoLedgerStorage { + // Get the XDR, regardless of ttl + fn get_xdr_internal(&self, key_xdr: &mut Vec) -> std::result::Result, Error> { + let key_c_xdr = CXDR { + xdr: key_xdr.as_mut_ptr(), + len: key_xdr.len(), + }; + let res = unsafe { SnapshotSourceGet(self.golang_handle, key_c_xdr) }; + if res.xdr.is_null() { + return Err(Error::NotFound); + } + let v = from_c_xdr(res); + unsafe { FreeGoXDR(res) }; + Ok(v) + } +} + +impl LedgerGetter for GoLedgerStorage { + fn get( + &self, + key: &LedgerKey, + include_not_live: bool, + ) -> std::result::Result<(LedgerEntry, Option), Error> { + let mut key_xdr = key.to_xdr(Limits::none())?; + let xdr = self.get_xdr_internal(&mut key_xdr)?; + + let live_until_ledger_seq = match key { + // TODO: it would probably be more efficient to do all of this in the Go side + // (e.g. it would allow us to query multiple entries at once) + LedgerKey::ContractData(_) | LedgerKey::ContractCode(_) => { + let key_hash: [u8; 32] = Sha256::digest(key_xdr).into(); + let ttl_key = LedgerKey::Ttl(LedgerKeyTtl { + key_hash: Hash(key_hash), + }); + let mut ttl_key_xdr = ttl_key.to_xdr(Limits::none())?; + let ttl_entry_xdr = self.get_xdr_internal(&mut ttl_key_xdr)?; + let ttl_entry = LedgerEntry::from_xdr(ttl_entry_xdr, Limits::none())?; + if let LedgerEntryData::Ttl(TtlEntry { + live_until_ledger_seq, + .. + }) = ttl_entry.data + { + Some(live_until_ledger_seq) + } else { + return Err(Error::UnexpectedLedgerEntryTypeForTtlKey { + ledger_entry_type: ttl_entry.data.name().to_string(), + }); + } + } + _ => None, + }; + + if !include_not_live + && live_until_ledger_seq.is_some() + && !is_live(live_until_ledger_seq.unwrap(), self.current_ledger_sequence) + { + return Err(Error::NotLive); + } + + let entry = LedgerEntry::from_xdr(xdr, Limits::none())?; + Ok((entry, live_until_ledger_seq)) + } +} + +pub(crate) fn is_live(live_until_ledger_seq: u32, current_ledger_seq: u32) -> bool { + live_until_ledger_seq >= current_ledger_seq +} diff --git a/cmd/soroban-rpc/lib/preflight/src/state_ttl.rs b/cmd/soroban-rpc/lib/preflight/src/state_ttl.rs index 5373177c91..a3ef4e6882 100644 --- a/cmd/soroban-rpc/lib/preflight/src/state_ttl.rs +++ b/cmd/soroban-rpc/lib/preflight/src/state_ttl.rs @@ -8,7 +8,7 @@ pub(crate) trait TTLLedgerEntry { fn durability(&self) -> ContractDataDurability; fn live_until_ledger_seq(&self) -> u32; fn is_live(&self, current_ledger_seq: u32) -> bool { - is_live(self.live_until_ledger_seq(), current_ledger_seq) + self.live_until_ledger_seq() >= current_ledger_seq } } @@ -55,10 +55,6 @@ impl<'a> TryInto> for &'a (LedgerEntry, Option } } -pub(crate) fn is_live(live_until_ledger_seq: u32, current_ledger_seq: u32) -> bool { - live_until_ledger_seq >= current_ledger_seq -} - pub(crate) fn get_restored_ledger_sequence( current_ledger_seq: u32, min_persistent_ttl: u32, diff --git a/scripts/check-dependencies.bash b/scripts/check-dependencies.bash index 470ac65ec0..7415e3950c 100755 --- a/scripts/check-dependencies.bash +++ b/scripts/check-dependencies.bash @@ -8,7 +8,6 @@ if [ -z "$(sed --version 2>&1 | grep GNU)" ]; then fi CURL="curl -sL --fail-with-body" -CARGO_PACKAGE_REVISION_EXTRACT_SED_COMMAND='s/.*rev=\(.*\)#.*/\1/' if ! CARGO_OUTPUT=$(cargo tree -p soroban-env-host 2>&1); then echo "The project depends on multiple versions of the soroban-env-host Rust library, please unify them." @@ -31,9 +30,28 @@ RS_STELLAR_XDR_REVISION="" # revision of https://github.com/stellar/stellar-xdr/ used by the Rust code STELLAR_XDR_REVISION_FROM_RUST="" +function stellar_xdr_version_from_rust_dep_tree { + LINE=$(grep stellar-xdr | head -n 1) + # try to obtain a commit + COMMIT=$(echo $LINE | $SED -n 's/.*rev=\(.*\)#.*/\1/p') + if [ -n "$COMMIT" ]; then + echo "$COMMIT" + return + fi + # obtain a crate version + echo $LINE | $SED -n 's/.*stellar-xdr \(v\)\{0,1\}\([^ ]*\).*/\2/p' +} + if CARGO_OUTPUT=$(cargo tree --depth 0 -p stellar-xdr 2>&1); then - RS_STELLAR_XDR_REVISION=$(echo $CARGO_OUTPUT | head -n 1 | $SED "$CARGO_PACKAGE_REVISION_EXTRACT_SED_COMMAND") - STELLAR_XDR_REVISION_FROM_RUST=$($CURL https://raw.githubusercontent.com/stellar/rs-stellar-xdr/${RS_STELLAR_XDR_REVISION}/xdr/curr-version) + RS_STELLAR_XDR_REVISION=$(echo "$CARGO_OUTPUT" | stellar_xdr_version_from_rust_dep_tree) + if [ ${#RS_STELLAR_XDR_REVISION} -eq 40 ]; then + # revision is a git hash + STELLAR_XDR_REVISION_FROM_RUST=$($CURL https://raw.githubusercontent.com/stellar/rs-stellar-xdr/${RS_STELLAR_XDR_REVISION}/xdr/curr-version) + else + # revision is a crate version + CARGO_SRC_BASE_DIR=$(realpath ${CARGO_HOME:-$HOME/.cargo}/registry/src/index*) + STELLAR_XDR_REVISION_FROM_RUST=$(cat "${CARGO_SRC_BASE_DIR}/stellar-xdr-${RS_STELLAR_XDR_REVISION}/xdr/curr-version") + fi else echo "The project depends on multiple versions of the Rust rs-stellar-xdr library" echo "Make sure a single version of stellar-xdr is used" @@ -81,7 +99,7 @@ fi CORE_HOST_DEP_TREE_CURR=$($CURL https://raw.githubusercontent.com/stellar/stellar-core/${CORE_CONTAINER_REVISION}/src/rust/src/host-dep-tree-curr.txt) -RS_STELLAR_XDR_REVISION_FROM_CORE=$(echo "$CORE_HOST_DEP_TREE_CURR" | grep stellar-xdr | head -n 1 | $SED "$CARGO_PACKAGE_REVISION_EXTRACT_SED_COMMAND") +RS_STELLAR_XDR_REVISION_FROM_CORE=$(echo "$CORE_HOST_DEP_TREE_CURR" | stellar_xdr_version_from_rust_dep_tree) if [ "$RS_STELLAR_XDR_REVISION" != "$RS_STELLAR_XDR_REVISION_FROM_CORE" ]; then echo "The Core revision used in integration tests (${CORE_CONTAINER_REVISION}) uses a different revision of https://github.com/stellar/rs-stellar-xdr" echo