Skip to content

Commit

Permalink
Fix dependency-check script to support crate versions (stellar#1129)
Browse files Browse the repository at this point in the history
  • Loading branch information
@2opremio
2opremio committed Dec 14, 2023
1 parent 397cc90 commit 6a0a621
Show file tree
Hide file tree
Showing 5 changed files with 153 additions and 98 deletions.
2 changes: 1 addition & 1 deletion cmd/soroban-rpc/lib/preflight/src/fees.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use anyhow::{bail, ensure, Context, Error, Result};
use ledger_storage::LedgerStorage;
use ledger_storage::{LedgerGetter, LedgerStorage};
use soroban_env_host::budget::Budget;
use soroban_env_host::e2e_invoke::{
extract_rent_changes, get_ledger_changes, LedgerEntryChange, TtlEntryMap,
Expand Down
116 changes: 33 additions & 83 deletions cmd/soroban-rpc/lib/preflight/src/ledger_storage.rs
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,17 @@
use sha2::Digest;
use soroban_env_host::storage::SnapshotSource;
use soroban_env_host::xdr::ContractDataDurability::{Persistent, Temporary};
use soroban_env_host::xdr::{
ConfigSettingEntry, ConfigSettingId, Error as XdrError, Hash, LedgerEntry, LedgerEntryData,
LedgerKey, LedgerKeyConfigSetting, LedgerKeyTtl, Limits, ReadXdr, ScError, ScErrorCode,
TtlEntry, WriteXdr,
ConfigSettingEntry, ConfigSettingId, Error as XdrError, LedgerEntry, LedgerEntryData,
LedgerKey, LedgerKeyConfigSetting, Limits, ScError, ScErrorCode, WriteXdr,
};
use soroban_env_host::HostError;
use state_ttl::{get_restored_ledger_sequence, is_live, TTLLedgerEntry};
use state_ttl::{get_restored_ledger_sequence, TTLLedgerEntry};
use std::cell::RefCell;
use std::collections::HashSet;
use std::convert::TryInto;
use std::ffi::NulError;
use std::rc::Rc;
use std::str::Utf8Error;
use {from_c_xdr, CXDR};

// Functions imported from Golang
extern "C" {
// Free Strings returned from Go functions
fn FreeGoXDR(xdr: CXDR);
// LedgerKey XDR in base64 string to LedgerEntry XDR in base64 string
fn SnapshotSourceGet(handle: libc::uintptr_t, ledger_key: CXDR) -> CXDR;
}

#[derive(thiserror::Error, Debug)]
pub(crate) enum Error {
Expand Down Expand Up @@ -88,28 +77,36 @@ impl EntryRestoreTracker {
}
}

pub(crate) trait LedgerGetter {
fn get(
&self,
key: &LedgerKey,
include_not_live: bool,
) -> anyhow::Result<(LedgerEntry, Option<u32>), Error>;
}

pub(crate) struct LedgerStorage {
golang_handle: libc::uintptr_t,
ledger_getter: Box<dyn LedgerGetter>,
current_ledger_sequence: u32,
restore_tracker: Option<EntryRestoreTracker>,
}

impl LedgerStorage {
pub(crate) fn new(golang_handle: libc::uintptr_t, current_ledger_sequence: u32) -> Self {
pub(crate) fn new(ledger_getter: Box<dyn LedgerGetter>, current_ledger_sequence: u32) -> Self {
LedgerStorage {
golang_handle,
ledger_getter,
current_ledger_sequence,
restore_tracker: None,
}
}

pub(crate) fn with_restore_tracking(
golang_handle: libc::uintptr_t,
ledger_getter: Box<dyn LedgerGetter>,
current_ledger_sequence: u32,
) -> Result<Self, Error> {
// First, we initialize it without the tracker, to get the minimum restore ledger from the network
let mut ledger_storage = LedgerStorage {
golang_handle,
ledger_getter,
current_ledger_sequence,
restore_tracker: None,
};
Expand All @@ -129,74 +126,14 @@ impl LedgerStorage {
Ok(ledger_storage)
}

// Get the XDR, regardless of ttl
fn get_xdr_internal(&self, key_xdr: &mut Vec<u8>) -> Result<Vec<u8>, Error> {
let key_c_xdr = CXDR {
xdr: key_xdr.as_mut_ptr(),
len: key_xdr.len(),
};
let res = unsafe { SnapshotSourceGet(self.golang_handle, key_c_xdr) };
if res.xdr.is_null() {
return Err(Error::NotFound);
}
let v = from_c_xdr(res);
unsafe { FreeGoXDR(res) };
Ok(v)
}

pub(crate) fn get(
&self,
key: &LedgerKey,
include_not_live: bool,
) -> Result<(LedgerEntry, Option<u32>), Error> {
let mut key_xdr = key.to_xdr(Limits::none())?;
let xdr = self.get_xdr_internal(&mut key_xdr)?;

let live_until_ledger_seq = match key {
// TODO: it would probably be more efficient to do all of this in the Go side
// (e.g. it would allow us to query multiple entries at once)
LedgerKey::ContractData(_) | LedgerKey::ContractCode(_) => {
let key_hash: [u8; 32] = sha2::Sha256::digest(key_xdr).into();
let ttl_key = LedgerKey::Ttl(LedgerKeyTtl {
key_hash: Hash(key_hash),
});
let mut ttl_key_xdr = ttl_key.to_xdr(Limits::none())?;
let ttl_entry_xdr = self.get_xdr_internal(&mut ttl_key_xdr)?;
let ttl_entry = LedgerEntry::from_xdr(ttl_entry_xdr, Limits::none())?;
if let LedgerEntryData::Ttl(TtlEntry {
live_until_ledger_seq,
..
}) = ttl_entry.data
{
Some(live_until_ledger_seq)
} else {
return Err(Error::UnexpectedLedgerEntryTypeForTtlKey {
ledger_entry_type: ttl_entry.data.name().to_string(),
});
}
}
_ => None,
};

if !include_not_live
&& live_until_ledger_seq.is_some()
&& !is_live(live_until_ledger_seq.unwrap(), self.current_ledger_sequence)
{
return Err(Error::NotLive);
}

let entry = LedgerEntry::from_xdr(xdr, Limits::none())?;
Ok((entry, live_until_ledger_seq))
}

pub(crate) fn get_xdr(
&self,
key: &LedgerKey,
include_not_live: bool,
) -> Result<Vec<u8>, Error> {
// TODO: this can be optimized since for entry types other than ContractCode/ContractData,
// they don't need to be deserialized and serialized again
let (entry, _) = self.get(key, include_not_live)?;
let (entry, _) = self.ledger_getter.get(key, include_not_live)?;
Ok(entry.to_xdr(Limits::none())?)
}

Expand All @@ -207,7 +144,7 @@ impl LedgerStorage {
let key = LedgerKey::ConfigSetting(LedgerKeyConfigSetting {
config_setting_id: setting_id,
});
match self.get(&key, false)? {
match self.ledger_getter.get(&key, false)? {
(
LedgerEntry {
data: LedgerEntryData::ConfigSetting(cs),
Expand All @@ -229,10 +166,20 @@ impl LedgerStorage {
}
}

impl LedgerGetter for LedgerStorage {
fn get(
&self,
key: &LedgerKey,
include_not_live: bool,
) -> anyhow::Result<(LedgerEntry, Option<u32>), Error> {
self.ledger_getter.get(key, include_not_live)
}
}

impl SnapshotSource for LedgerStorage {
fn get(&self, key: &Rc<LedgerKey>) -> Result<(Rc<LedgerEntry>, Option<u32>), HostError> {
if let Some(ref tracker) = self.restore_tracker {
let mut entry_and_ttl = self.get(key, true)?;
let mut entry_and_ttl = self.ledger_getter.get(key, true)?;
// Explicitly discard temporary ttl'ed entries
if let Ok(ttl_entry) = TryInto::<Box<dyn TTLLedgerEntry>>::try_into(&entry_and_ttl) {
if ttl_entry.durability() == Temporary
Expand All @@ -246,7 +193,10 @@ impl SnapshotSource for LedgerStorage {
tracker.track_and_restore(self.current_ledger_sequence, key, &entry_and_ttl);
return Ok((entry_and_ttl.0.into(), entry_and_ttl.1));
}
let entry_and_ttl = <LedgerStorage>::get(self, key, false).map_err(HostError::from)?;
let entry_and_ttl = self
.ledger_getter
.get(key, false)
.map_err(HostError::from)?;
Ok((entry_and_ttl.0.into(), entry_and_ttl.1))
}

Expand Down
101 changes: 96 additions & 5 deletions cmd/soroban-rpc/lib/preflight/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,12 @@ extern crate sha2;
extern crate soroban_env_host;

use anyhow::{Context, Result};
use ledger_storage::LedgerStorage;
use ledger_storage::{Error, LedgerGetter, LedgerStorage};
use preflight::PreflightResult;
use sha2::{Digest, Sha256};
use soroban_env_host::xdr::{
AccountId, InvokeHostFunctionOp, LedgerFootprint, Limits, OperationBody, ReadXdr, WriteXdr,
AccountId, Hash, InvokeHostFunctionOp, LedgerEntry, LedgerEntryData, LedgerFootprint,
LedgerKey, LedgerKeyTtl, Limits, OperationBody, ReadXdr, TtlEntry, WriteXdr,
};
use soroban_env_host::LedgerInfo;
use std::ffi::{CStr, CString};
Expand Down Expand Up @@ -158,8 +159,13 @@ fn preflight_invoke_hf_op_or_maybe_panic(
let invoke_hf_op =
InvokeHostFunctionOp::from_xdr(from_c_xdr(invoke_hf_op), Limits::none()).unwrap();
let source_account = AccountId::from_xdr(from_c_xdr(source_account), Limits::none()).unwrap();
let ledger_storage = LedgerStorage::with_restore_tracking(handle, ledger_info.sequence_number)
.context("cannot create LedgerStorage")?;
let go_storage = GoLedgerStorage {
golang_handle: handle,
current_ledger_sequence: ledger_info.sequence_number,
};
let ledger_storage =
LedgerStorage::with_restore_tracking(Box::new(go_storage), ledger_info.sequence_number)
.context("cannot create LedgerStorage")?;
let result = preflight::preflight_invoke_hf_op(
ledger_storage,
bucket_list_size,
Expand Down Expand Up @@ -199,7 +205,11 @@ fn preflight_footprint_ttl_op_or_maybe_panic(
) -> Result<CPreflightResult> {
let op_body = OperationBody::from_xdr(from_c_xdr(op_body), Limits::none()).unwrap();
let footprint = LedgerFootprint::from_xdr(from_c_xdr(footprint), Limits::none()).unwrap();
let ledger_storage = &LedgerStorage::new(handle, current_ledger_seq);
let go_storage = GoLedgerStorage {
golang_handle: handle,
current_ledger_sequence: current_ledger_seq,
};
let ledger_storage = &LedgerStorage::new(Box::new(go_storage), current_ledger_seq);
let result = preflight::preflight_footprint_ttl_op(
ledger_storage,
bucket_list_size,
Expand Down Expand Up @@ -342,3 +352,84 @@ fn from_c_xdr(xdr: CXDR) -> Vec<u8> {
let s = unsafe { slice::from_raw_parts(xdr.xdr, xdr.len) };
s.to_vec()
}

// Functions imported from Golang
extern "C" {
// Free Strings returned from Go functions
fn FreeGoXDR(xdr: CXDR);
// LedgerKey XDR in base64 string to LedgerEntry XDR in base64 string
fn SnapshotSourceGet(handle: libc::uintptr_t, ledger_key: CXDR) -> CXDR;
}

struct GoLedgerStorage {
golang_handle: libc::uintptr_t,
current_ledger_sequence: u32,
}

impl GoLedgerStorage {
// Get the XDR, regardless of ttl
fn get_xdr_internal(&self, key_xdr: &mut Vec<u8>) -> std::result::Result<Vec<u8>, Error> {
let key_c_xdr = CXDR {
xdr: key_xdr.as_mut_ptr(),
len: key_xdr.len(),
};
let res = unsafe { SnapshotSourceGet(self.golang_handle, key_c_xdr) };
if res.xdr.is_null() {
return Err(Error::NotFound);
}
let v = from_c_xdr(res);
unsafe { FreeGoXDR(res) };
Ok(v)
}
}

impl LedgerGetter for GoLedgerStorage {
fn get(
&self,
key: &LedgerKey,
include_not_live: bool,
) -> std::result::Result<(LedgerEntry, Option<u32>), Error> {
let mut key_xdr = key.to_xdr(Limits::none())?;
let xdr = self.get_xdr_internal(&mut key_xdr)?;

let live_until_ledger_seq = match key {
// TODO: it would probably be more efficient to do all of this in the Go side
// (e.g. it would allow us to query multiple entries at once)
LedgerKey::ContractData(_) | LedgerKey::ContractCode(_) => {
let key_hash: [u8; 32] = Sha256::digest(key_xdr).into();
let ttl_key = LedgerKey::Ttl(LedgerKeyTtl {
key_hash: Hash(key_hash),
});
let mut ttl_key_xdr = ttl_key.to_xdr(Limits::none())?;
let ttl_entry_xdr = self.get_xdr_internal(&mut ttl_key_xdr)?;
let ttl_entry = LedgerEntry::from_xdr(ttl_entry_xdr, Limits::none())?;
if let LedgerEntryData::Ttl(TtlEntry {
live_until_ledger_seq,
..
}) = ttl_entry.data
{
Some(live_until_ledger_seq)
} else {
return Err(Error::UnexpectedLedgerEntryTypeForTtlKey {
ledger_entry_type: ttl_entry.data.name().to_string(),
});
}
}
_ => None,
};

if !include_not_live
&& live_until_ledger_seq.is_some()
&& !is_live(live_until_ledger_seq.unwrap(), self.current_ledger_sequence)
{
return Err(Error::NotLive);
}

let entry = LedgerEntry::from_xdr(xdr, Limits::none())?;
Ok((entry, live_until_ledger_seq))
}
}

pub(crate) fn is_live(live_until_ledger_seq: u32, current_ledger_seq: u32) -> bool {
live_until_ledger_seq >= current_ledger_seq
}
6 changes: 1 addition & 5 deletions cmd/soroban-rpc/lib/preflight/src/state_ttl.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ pub(crate) trait TTLLedgerEntry {
fn durability(&self) -> ContractDataDurability;
fn live_until_ledger_seq(&self) -> u32;
fn is_live(&self, current_ledger_seq: u32) -> bool {
is_live(self.live_until_ledger_seq(), current_ledger_seq)
self.live_until_ledger_seq() >= current_ledger_seq
}
}

Expand Down Expand Up @@ -55,10 +55,6 @@ impl<'a> TryInto<Box<dyn TTLLedgerEntry + 'a>> for &'a (LedgerEntry, Option<u32>
}
}

pub(crate) fn is_live(live_until_ledger_seq: u32, current_ledger_seq: u32) -> bool {
live_until_ledger_seq >= current_ledger_seq
}

pub(crate) fn get_restored_ledger_sequence(
current_ledger_seq: u32,
min_persistent_ttl: u32,
Expand Down
26 changes: 22 additions & 4 deletions scripts/check-dependencies.bash
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ if [ -z "$(sed --version 2>&1 | grep GNU)" ]; then
fi

CURL="curl -sL --fail-with-body"
CARGO_PACKAGE_REVISION_EXTRACT_SED_COMMAND='s/.*rev=\(.*\)#.*/\1/'

if ! CARGO_OUTPUT=$(cargo tree -p soroban-env-host 2>&1); then
echo "The project depends on multiple versions of the soroban-env-host Rust library, please unify them."
Expand All @@ -31,9 +30,28 @@ RS_STELLAR_XDR_REVISION=""
# revision of https://github.com/stellar/stellar-xdr/ used by the Rust code
STELLAR_XDR_REVISION_FROM_RUST=""

function stellar_xdr_version_from_rust_dep_tree {
LINE=$(grep stellar-xdr | head -n 1)
# try to obtain a commit
COMMIT=$(echo $LINE | $SED -n 's/.*rev=\(.*\)#.*/\1/p')
if [ -n "$COMMIT" ]; then
echo "$COMMIT"
return
fi
# obtain a crate version
echo $LINE | $SED -n 's/.*stellar-xdr \(v\)\{0,1\}\([^ ]*\).*/\2/p'
}

if CARGO_OUTPUT=$(cargo tree --depth 0 -p stellar-xdr 2>&1); then
RS_STELLAR_XDR_REVISION=$(echo $CARGO_OUTPUT | head -n 1 | $SED "$CARGO_PACKAGE_REVISION_EXTRACT_SED_COMMAND")
STELLAR_XDR_REVISION_FROM_RUST=$($CURL https://raw.githubusercontent.com/stellar/rs-stellar-xdr/${RS_STELLAR_XDR_REVISION}/xdr/curr-version)
RS_STELLAR_XDR_REVISION=$(echo "$CARGO_OUTPUT" | stellar_xdr_version_from_rust_dep_tree)
if [ ${#RS_STELLAR_XDR_REVISION} -eq 40 ]; then
# revision is a git hash
STELLAR_XDR_REVISION_FROM_RUST=$($CURL https://raw.githubusercontent.com/stellar/rs-stellar-xdr/${RS_STELLAR_XDR_REVISION}/xdr/curr-version)
else
# revision is a crate version
CARGO_SRC_BASE_DIR=$(realpath ${CARGO_HOME:-$HOME/.cargo}/registry/src/index*)
STELLAR_XDR_REVISION_FROM_RUST=$(cat "${CARGO_SRC_BASE_DIR}/stellar-xdr-${RS_STELLAR_XDR_REVISION}/xdr/curr-version")
fi
else
echo "The project depends on multiple versions of the Rust rs-stellar-xdr library"
echo "Make sure a single version of stellar-xdr is used"
Expand Down Expand Up @@ -81,7 +99,7 @@ fi
CORE_HOST_DEP_TREE_CURR=$($CURL https://raw.githubusercontent.com/stellar/stellar-core/${CORE_CONTAINER_REVISION}/src/rust/src/host-dep-tree-curr.txt)


RS_STELLAR_XDR_REVISION_FROM_CORE=$(echo "$CORE_HOST_DEP_TREE_CURR" | grep stellar-xdr | head -n 1 | $SED "$CARGO_PACKAGE_REVISION_EXTRACT_SED_COMMAND")
RS_STELLAR_XDR_REVISION_FROM_CORE=$(echo "$CORE_HOST_DEP_TREE_CURR" | stellar_xdr_version_from_rust_dep_tree)
if [ "$RS_STELLAR_XDR_REVISION" != "$RS_STELLAR_XDR_REVISION_FROM_CORE" ]; then
echo "The Core revision used in integration tests (${CORE_CONTAINER_REVISION}) uses a different revision of https://github.com/stellar/rs-stellar-xdr"
echo
Expand Down

0 comments on commit 6a0a621

Please sign in to comment.