Skip to content

Commit

Permalink
Enforce tx metadata upload with static types (#23028)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jstarry
jstarry authored Feb 10, 2022
1 parent 2ca7ae8 commit d5dec98
Show file tree
Hide file tree
Showing 16 changed files with 480 additions and 438 deletions.
23 changes: 9 additions & 14 deletions client-test/tests/client.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ use {
RpcAccountInfoConfig, RpcBlockSubscribeConfig, RpcBlockSubscribeFilter,
RpcProgramAccountsConfig,
},
rpc_response::{RpcBlockUpdate, SlotInfo},
rpc_response::SlotInfo,
},
solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path},
solana_rpc::{
Expand All @@ -36,7 +36,7 @@ use {
},
solana_streamer::socket::SocketAddrSpace,
solana_test_validator::TestValidator,
solana_transaction_status::{TransactionDetails, UiTransactionEncoding},
solana_transaction_status::{ConfirmedBlock, TransactionDetails, UiTransactionEncoding},
std::{
collections::HashSet,
net::{IpAddr, SocketAddr},
Expand Down Expand Up @@ -214,6 +214,7 @@ fn test_block_subscription() {
..
} = create_genesis_config(10_000);
let bank = Bank::new_for_tests(&genesis_config);
let rent_exempt_amount = bank.get_minimum_balance_for_rent_exemption(0);
let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));

// setup Blockstore
Expand All @@ -227,6 +228,8 @@ fn test_block_subscription() {
let keypair2 = Keypair::new();
let keypair3 = Keypair::new();
let max_complete_transaction_status_slot = Arc::new(AtomicU64::new(blockstore.max_root()));
bank.transfer(rent_exempt_amount, &alice, &keypair2.pubkey())
.unwrap();
let _confirmed_block_signatures = create_test_transactions_and_populate_blockstore(
vec![&alice, &keypair1, &keypair2, &keypair3],
0,
Expand Down Expand Up @@ -278,23 +281,15 @@ fn test_block_subscription() {
match maybe_actual {
Ok(actual) => {
let versioned_block = blockstore.get_complete_block(slot, false).unwrap();
let legacy_block = versioned_block.into_legacy_block().unwrap();
let block = legacy_block.clone().configure(
UiTransactionEncoding::Json,
TransactionDetails::Signatures,
false,
);
let expected = RpcBlockUpdate {
slot,
block: Some(block),
err: None,
};
let legacy_block = ConfirmedBlock::from(versioned_block)
.into_legacy_block()
.unwrap();
let block = legacy_block.configure(
UiTransactionEncoding::Json,
TransactionDetails::Signatures,
false,
);
assert_eq!(actual.value.slot, expected.slot);
assert_eq!(actual.value.slot, slot);
assert!(block.eq(&actual.value.block.unwrap()));
}
Err(e) => {
Expand Down
54 changes: 22 additions & 32 deletions core/src/banking_stage.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2955,16 +2955,9 @@ mod tests {
);
let ix_error_signature = ix_error_tx.signatures[0];
let entry_2 = next_entry(&entry_1.hash, 1, vec![ix_error_tx.clone()]);
let fail_tx = system_transaction::transfer(
&mint_keypair,
&pubkey1,
rent_exempt_amount,
genesis_config.hash(),
);
let entry_3 = next_entry(&entry_2.hash, 1, vec![fail_tx.clone()]);
let entries = vec![entry_1, entry_2, entry_3];
let entries = vec![entry_1, entry_2];

let transactions = sanitize_transactions(vec![success_tx, ix_error_tx, fail_tx]);
let transactions = sanitize_transactions(vec![success_tx, ix_error_tx]);
bank.transfer(rent_exempt_amount, &mint_keypair, &keypair1.pubkey())
.unwrap();

Expand Down Expand Up @@ -3024,27 +3017,24 @@ mod tests {
transaction_status_service.join().unwrap();

let confirmed_block = blockstore.get_rooted_block(bank.slot(), false).unwrap();
assert_eq!(confirmed_block.transactions.len(), 3);

for VersionedTransactionWithStatusMeta { transaction, meta } in
confirmed_block.transactions.into_iter()
{
if transaction.signatures[0] == success_signature {
let meta = meta.unwrap();
assert_eq!(meta.status, Ok(()));
} else if transaction.signatures[0] == ix_error_signature {
let meta = meta.unwrap();
assert_eq!(
meta.status,
Err(TransactionError::InstructionError(
0,
InstructionError::Custom(1)
))
);
} else {
assert_eq!(meta, None);
}
}
let actual_tx_results: Vec<_> = confirmed_block
.transactions
.into_iter()
.map(|VersionedTransactionWithStatusMeta { transaction, meta }| {
(transaction.signatures[0], meta.status)
})
.collect();
let expected_tx_results = vec![
(success_signature, Ok(())),
(
ix_error_signature,
Err(TransactionError::InstructionError(
0,
InstructionError::Custom(1),
)),
),
];
assert_eq!(actual_tx_results, expected_tx_results);

poh_recorder
.lock()
Expand Down Expand Up @@ -3191,7 +3181,7 @@ mod tests {
let recorded_meta = confirmed_block.transactions.pop().unwrap().meta;
assert_eq!(
recorded_meta,
Some(TransactionStatusMeta {
TransactionStatusMeta {
status: Ok(()),
pre_balances: vec![1, 0, 0],
post_balances: vec![1, 0, 0],
Expand All @@ -3200,7 +3190,7 @@ mod tests {
rewards: Some(vec![]),
loaded_addresses: sanitized_tx.get_loaded_addresses(),
..TransactionStatusMeta::default()
})
}
);
poh_recorder
.lock()
Expand Down
45 changes: 22 additions & 23 deletions core/src/replay_stage.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3984,36 +3984,35 @@ pub mod tests {
let bank1 = Arc::new(Bank::new_from_parent(&bank0, &Pubkey::default(), 1));
let slot = bank1.slot();

let signatures = create_test_transactions_and_populate_blockstore(
let mut test_signatures_iter = create_test_transactions_and_populate_blockstore(
vec![&mint_keypair, &keypair1, &keypair2, &keypair3],
bank0.slot(),
bank1,
blockstore.clone(),
Arc::new(AtomicU64::default()),
);
)
.into_iter();

let confirmed_block = blockstore.get_rooted_block(slot, false).unwrap();
assert_eq!(confirmed_block.transactions.len(), 3);

for VersionedTransactionWithStatusMeta { transaction, meta } in
confirmed_block.transactions.into_iter()
{
if transaction.signatures[0] == signatures[0] {
let meta = meta.unwrap();
assert_eq!(meta.status, Ok(()));
} else if transaction.signatures[0] == signatures[1] {
let meta = meta.unwrap();
assert_eq!(
meta.status,
Err(TransactionError::InstructionError(
0,
InstructionError::Custom(1)
))
);
} else {
assert_eq!(meta, None);
}
}
let actual_tx_results: Vec<_> = confirmed_block
.transactions
.into_iter()
.map(|VersionedTransactionWithStatusMeta { transaction, meta }| {
(transaction.signatures[0], meta.status)
})
.collect();
let expected_tx_results = vec![
(test_signatures_iter.next().unwrap(), Ok(())),
(
test_signatures_iter.next().unwrap(),
Err(TransactionError::InstructionError(
0,
InstructionError::Custom(1),
)),
),
];
assert_eq!(actual_tx_results, expected_tx_results);
assert!(test_signatures_iter.next().is_none());
}
Blockstore::destroy(&ledger_path).unwrap();
}
Expand Down
40 changes: 15 additions & 25 deletions ledger-tool/src/bigtable.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ use {
},
solana_ledger::{blockstore::Blockstore, blockstore_db::AccessType},
solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature},
solana_transaction_status::{ConfirmedBlock, Encodable, UiTransactionEncoding},
solana_transaction_status::{Encodable, LegacyConfirmedBlock, UiTransactionEncoding},
std::{
collections::HashSet,
path::Path,
Expand All @@ -30,7 +30,6 @@ async fn upload(
blockstore: Blockstore,
starting_slot: Slot,
ending_slot: Option<Slot>,
allow_missing_metadata: bool,
force_reupload: bool,
) -> Result<(), Box<dyn std::error::Error>> {
let bigtable = solana_storage_bigtable::LedgerStorage::new(false, None, None)
Expand All @@ -42,7 +41,6 @@ async fn upload(
bigtable,
starting_slot,
ending_slot,
allow_missing_metadata,
force_reupload,
Arc::new(AtomicBool::new(false)),
)
Expand Down Expand Up @@ -73,13 +71,13 @@ async fn block(slot: Slot, output_format: OutputFormat) -> Result<(), Box<dyn st
.await
.map_err(|err| format!("Failed to connect to storage: {:?}", err))?;

let versioned_block = bigtable.get_confirmed_block(slot).await?;
let block = versioned_block
let confirmed_block = bigtable.get_confirmed_block(slot).await?;
let legacy_block = confirmed_block
.into_legacy_block()
.ok_or_else(|| "Failed to read versioned transaction in block".to_string())?;

let cli_block = CliBlock {
encoded_confirmed_block: block.encode(UiTransactionEncoding::Base64),
encoded_confirmed_block: legacy_block.encode(UiTransactionEncoding::Base64),
slot,
};
println!("{}", output_format.formatted_string(&cli_block));
Expand Down Expand Up @@ -156,20 +154,20 @@ async fn confirm(
let mut get_transaction_error = None;
if verbose {
match bigtable.get_confirmed_transaction(signature).await {
Ok(Some(versioned_confirmed_tx)) => {
let confirmed_transaction = versioned_confirmed_tx
Ok(Some(confirmed_tx)) => {
let legacy_confirmed_tx = confirmed_tx
.into_legacy_confirmed_transaction()
.ok_or_else(|| "Failed to read versioned transaction in block".to_string())?;

transaction = Some(CliTransaction {
transaction: confirmed_transaction
.transaction
transaction: legacy_confirmed_tx
.tx_with_meta
.transaction
.encode(UiTransactionEncoding::Json),
meta: confirmed_transaction.transaction.meta.map(|m| m.into()),
block_time: confirmed_transaction.block_time,
slot: Some(confirmed_transaction.slot),
decoded_transaction: confirmed_transaction.transaction.transaction,
meta: legacy_confirmed_tx.tx_with_meta.meta.map(|m| m.into()),
block_time: legacy_confirmed_tx.block_time,
slot: Some(legacy_confirmed_tx.slot),
decoded_transaction: legacy_confirmed_tx.tx_with_meta.transaction,
prefix: " ".to_string(),
sigverify_status: vec![],
});
Expand Down Expand Up @@ -201,7 +199,7 @@ pub async fn transaction_history(
) -> Result<(), Box<dyn std::error::Error>> {
let bigtable = solana_storage_bigtable::LedgerStorage::new(true, None, None).await?;

let mut loaded_block: Option<(Slot, ConfirmedBlock)> = None;
let mut loaded_block: Option<(Slot, LegacyConfirmedBlock)> = None;
while limit > 0 {
let results = bigtable
.get_confirmed_signatures_for_address(
Expand Down Expand Up @@ -267,8 +265,8 @@ pub async fn transaction_history(
println!(" Unable to get confirmed transaction details: {}", err);
break;
}
Ok(versioned_block) => {
let block = versioned_block.into_legacy_block().ok_or_else(|| {
Ok(confirmed_block) => {
let block = confirmed_block.into_legacy_block().ok_or_else(|| {
"Failed to read versioned transaction in block".to_string()
})?;
loaded_block = Some((result.slot, block));
Expand Down Expand Up @@ -316,12 +314,6 @@ impl BigTableSubCommand for App<'_, '_> {
.index(2)
.help("Stop uploading at this slot [default: last available slot]"),
)
.arg(
Arg::with_name("allow_missing_metadata")
.long("allow-missing-metadata")
.takes_value(false)
.help("Don't panic if transaction metadata is missing"),
)
.arg(
Arg::with_name("force_reupload")
.long("force")
Expand Down Expand Up @@ -516,7 +508,6 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
("upload", Some(arg_matches)) => {
let starting_slot = value_t!(arg_matches, "starting_slot", Slot).unwrap_or(0);
let ending_slot = value_t!(arg_matches, "ending_slot", Slot).ok();
let allow_missing_metadata = arg_matches.is_present("allow_missing_metadata");
let force_reupload = arg_matches.is_present("force_reupload");
let blockstore = crate::open_blockstore(
&canonicalize_ledger_path(ledger_path),
Expand All @@ -528,7 +519,6 @@ pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
blockstore,
starting_slot,
ending_slot,
allow_missing_metadata,
force_reupload,
))
}
Expand Down
16 changes: 1 addition & 15 deletions ledger/src/bigtable_upload.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ pub async fn upload_confirmed_blocks(
bigtable: solana_storage_bigtable::LedgerStorage,
starting_slot: Slot,
ending_slot: Option<Slot>,
allow_missing_metadata: bool,
force_reupload: bool,
exit: Arc<AtomicBool>,
) -> Result<(), Box<dyn std::error::Error>> {
Expand Down Expand Up @@ -187,20 +186,7 @@ pub async fn upload_confirmed_blocks(
num_blocks -= 1;
None
}
Some(confirmed_block) => {
if confirmed_block
.transactions
.iter()
.any(|transaction| transaction.meta.is_none())
{
if allow_missing_metadata {
info!("Transaction metadata missing from slot {}", slot);
} else {
panic!("Transaction metadata missing from slot {}", slot);
}
}
Some(bigtable.upload_confirmed_block(slot, confirmed_block))
}
Some(confirmed_block) => Some(bigtable.upload_confirmed_block(slot, confirmed_block)),
});

for result in futures::future::join_all(uploads).await {
Expand Down
1 change: 0 additions & 1 deletion ledger/src/bigtable_upload_service.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@ impl BigTableUploadService {
bigtable_ledger_storage.clone(),
start_slot,
Some(end_slot),
true,
false,
exit.clone(),
));
Expand Down
Loading

0 comments on commit d5dec98

Please sign in to comment.